Merge branch 'feature-34-test_framework' into develop

.gitlab-ci.yml

+job1:
+  script:
+   - echo $PWD
+   - echo $USER
+   - echo $OAI_USER
+   - echo $OAI_EXTRA_ARGS
+   - echo $NFS_SHARE_DIR
+   - echo $EXTERNAL_SHARE_DIR
+   - echo $SHELL
+   - git rev-parse --abbrev-ref HEAD
+   - git_head=`git rev-parse HEAD`
+   - echo $git_head
+   - tmp=`git show-ref --head | grep $git_head`
+   - tmp=(${tmp///// })
+   - git_branch=${tmp[@]:(-1)}
+   - echo $git_branch
+   - source oaienv
+   - echo $OPENAIR_DIR
+   - NFS_TEST_RESULTS_DIR=$NFS_SHARE_DIR/$git_branch/$git_head
+   - EXTERNAL_SHARE_RESULTS_DIR=$EXTERNAL_SHARE_DIR/$git_branch/$git_head
+   - echo $NFS_TEST_RESULTS_DIR
+   - echo $EXTERNAL_SHARE_RESULTS_DIR
+   - mkdir -p $OPENAIR_DIR/cmake_targets/autotests/log
+   - python $OPENAIR_DIR/cmake_targets/autotests/run_exec_lte-softmodem_tests.py -r -5GRepoHeadVersion $git_head -n $NFS_SHARE_DIR -u $OAI_USER -p $OAI_PASS `echo $OAI_EXTRA_ARGS` -g $OAI_TEST_CASE_GROUP >& $OPENAIR_DIR/cmake_targets/autotests/python_autotest.log
+   - mv $OPENAIR_DIR/cmake_targets/autotests/python_autotest.log $OPENAIR_DIR/cmake_targets/autotests/log/python_autotest.log
+   - sshpass -p "$OAI_PASS" rsync -az -e "ssh -o StrictHostKeyChecking=no "  --rsync-path="mkdir -p $NFS_TEST_RESULTS_DIR && rsync" $OPENAIR_DIR/cmake_targets/autotests/log $OAI_USER@localhost:$NFS_TEST_RESULTS_DIR
+   - sshpass -p "$OAI_PASS" rsync -az -e "ssh -o StrictHostKeyChecking=no "  --rsync-path="mkdir -p $EXTERNAL_SHARE_DIR && rsync" $OPENAIR_DIR/cmake_targets/autotests/log $OAI_USER@localhost:$EXTERNAL_SHARE_DIR
+   - cat $OPENAIR_DIR/cmake_targets/autotests/log/results_autotests.xml

cmake_targets/CMakeLists.txt

@@ -421,7 +421,7 @@ elseif (${RF_BOARD} STREQUAL "OAI_USRP")
     ${OPENAIR_TARGETS}/ARCH/USRP/USERSPACE/LIB/usrp_lib.cpp)
   LINK_DIRECTORIES("/opt/lib")
   set(option_HW_lib "uhd")
-  set(LOWLATENCY False)
+  #set(LOWLATENCY False)
 
 elseif (${RF_BOARD} STREQUAL "OAI_BLADERF")
   include_directories("${OPENAIR_TARGETS}/ARCH/BLADERF/USERSPACE/LIB/")
@@ -432,14 +432,14 @@ elseif (${RF_BOARD} STREQUAL "OAI_BLADERF")
     )
   LINK_DIRECTORIES("/usr/lib/x86_64-linux-gnu")
   set(option_HW_lib "bladeRF")
-#set(LOWLATENCY False)
+ #set(LOWLATENCY False)
 
 elseif (${RF_BOARD} STREQUAL "ETHERNET")
   include_directories ("${OPENAIR_TARGETS}/ARCH/ETHERNET/USERSPACE/LIB")
   set(HW_SOURCE ${HW_SOURCE}
     ${OPENAIR_TARGETS}/ARCH/ETHERNET/USERSPACE/LIB/ethernet_lib.c
     )
-  set(LOWLATENCY True)
+  #set(LOWLATENCY True)
   
 elseif (${RF_BOARD} STREQUAL "CPRIGW")
   set(HW_SOURCE ${HW_SOURCE}
@@ -451,6 +451,7 @@ endif (${RF_BOARD} STREQUAL "EXMIMO")
 
 include_directories ("${OPENAIR_TARGETS}/ARCH/COMMON")
 
+Message("LOWLATENCY flag  is ${LOWLATENCY}")
 
 ##############################################################
 #    ???!!! TO BE DOCUMENTED OPTIONS !!!???

cmake_targets/autotests/core.py

@@ -118,8 +118,7 @@ class core:
         rsp1 = self.prompt1
         rsp2 = self.prompt2
         if sudo == True:
-          command = 'echo \'' + self.password + '\' | sudo -S -E ' + command
-
+           command = 'echo \'' + '' + '\' | sudo -S -E bash -c \' ' + command + '\' '
         self.wait_quiet()
         if self.send_cr:
             log.stats['cmd'] += 1
@@ -140,7 +139,7 @@ class core:
             rsp2 = self.prompt2
         self.wait_quiet()
         if sudo == True:
-          command = 'echo \'' + self.password + '\' | sudo -S -E ' + command
+            command = 'echo \'' + '' + '\' | sudo -S -E bash -c \' ' + command + '\' '
         if self.send_cr:
             log.stats['cmd'] += 1
             self.oai.sendline(command)
@@ -240,7 +239,8 @@ class core:
     #    print rsp
         if  (rsp.find(expect) == -1):
             return 'OK'
-
+        else:
+            print "command = " + cmd + "expect  = " + expect + "rsp = " + rsp
             raise log.err(self.failed(command, expect,debug))
 
     

cmake_targets/autotests/openair.py

@@ -83,10 +83,13 @@ class openair(core):
         return (stdout, stderr)
 
     def connect(self, username, password, prompt='PEXPECT_OAI'):
-     while True:  
+     max_retries=100
+     i=0
+     while i <= max_retries:  
         self.prompt1 = prompt
         self.prompt2 = prompt
         self.password = '' 
+        i=i+1
         # WE do not store the password when sending commands for secuirity reasons. The password might be accidentally logged in such cases.
         #The password is used only to make ssh connections. In case user wants to run programs with sudo, then he/she needs to add following line in /etc/sudoers
         # your_user_name  ALL=(ALL:ALL) NOPASSWD: ALL
@@ -109,11 +112,14 @@ class openair(core):
         except Exception, e:
             error=''
             error = error + ' In function: ' + sys._getframe().f_code.co_name + ': *** Caught exception: '  + str(e.__class__) + " : " + str( e)
+            error = error + 'address = "'+ self.address +' username = ' + username
             error = error + traceback.format_exc()
             print error
-            print "Retrying again in 60 seconds"
-            time.sleep(60)
-            #sys.exit(1)
+            print "Retrying again in 1  seconds"
+            time.sleep(1)
+            if i==max_retries:
+              print "Fatal Error: Terminating the program now..."
+              sys.exit(1)
                 
     def connect2(self, username, password, prompt='$'):
         self.prompt1 = prompt

cmake_targets/autotests/tools/configure_cots_bandrich_ue.py

@@ -10,6 +10,7 @@ import threading
 import signal
 import traceback
 import os
+import commands
 
 # configure the serial connections (the parameters differs on the device you are connecting to)
 #First we find an open port to work with
@@ -34,8 +35,7 @@ def find_open_port():
    ser = serial.Serial(port=serial_port)
    return
 
-find_open_port()
-print 'Using Serial port : ' + serial_port  
+
     
 #serial_port = '/dev/ttyUSB2'
 bandrich_ppd_config = os.environ.get('OPENAIR_DIR') + '/cmake_targets/autotests/tools/wdial.bandrich.conf'
@@ -127,7 +127,7 @@ def start_ue () :
         ip = IPRoute()
         idx = ip.link_lookup(ifname=iface)[0]
         os.system ('route add 192.172.0.1 ppp0')
-        os.system ('ping -c 5 192.172.0.1')
+        os.system ('ping 192.172.0.1')
         break
      except Exception, e:
         error = ' Interface ' + iface + 'does not exist...'
@@ -144,11 +144,63 @@ def stop_ue():
    send_command('AT+CGATT=0' , 'OK|ERROR' , timeout)
    send_command('AT+CFUN=4' , 'OK' , timeout)
 
+
+#reset the USB BUS of Bandrich UE
+def reset_ue():
+  stringIdBandrich='BandRich, Inc. 4G LTE adapter'
+  status, out = commands.getstatusoutput('lsusb | grep -i \'' + stringIdBandrich + '\'')
+  if (out == '') :
+     print "Bandrich 4G LTE Adapter not found. Exiting now..."
+     sys.exit()
+  p=re.compile('Bus\s*(\w+)\s*Device\s*(\w+):\s*ID\s*(\w+):(\w+)')
+  res=p.findall(out)
+  BusId=res[0][0]
+  DeviceId=res[0][1]
+  VendorId=res[0][2]
+  ProductId=res[0][3]
+  usb_dir= find_usb_path(VendorId, ProductId)
+  print usb_dir
+  cmd = "sudo sh -c \"echo 0 > " + usb_dir + "/authorized\""
+  os.system(cmd + " ; sleep 5" )
+  cmd = "sudo sh -c \"echo 1 > " + usb_dir + "/authorized\""
+  os.system(cmd + " ; sleep 5" )
+
+def read_file(filename):
+  try:
+    file = open(filename, 'r')
+    return file.read()
+  except Exception, e:
+    #error = ' Filename ' + filename 
+    #error = error + ' In function: ' + sys._getframe().f_code.co_name + ': *** Caught exception: '  + str(e.__class__) + " : " + str( e)
+    #error = error + traceback.format_exc()
+    #print error
+    return ''
+
+
+def find_usb_path(idVendor, idProduct):
+  for root, dirs, files in os.walk("/sys/bus/usb/devices", topdown=False):
+    for name in dirs:
+        tmpdir= os.path.join(root, name)
+        tmpidVendor = read_file(tmpdir+'/idVendor').replace("\n","")
+        tmpidProduct = read_file(tmpdir+'/idProduct').replace("\n","")
+        #print "tmpdir = " + tmpdir + " tmpidVendor = " + tmpidVendor + " tmpidProduct = " + tmpidProduct
+        if tmpidVendor == idVendor and tmpidProduct == idProduct:
+            return tmpdir
+  return ''
+
 for arg in sys.argv[1:]:
     if arg == '--start-ue' :
+        find_open_port()
+        print 'Using Serial port : ' + serial_port  
         start_ue()
     elif arg == '--stop-ue' :
+        find_open_port()
+        print 'Using Serial port : ' + serial_port  
         stop_ue()
+    elif arg == '--reset-ue' :
+        reset_ue()
     else :
         print " Script called with wrong arguments, arg = " + arg
         sys.exit()
+
+

cmake_targets/autotests/tools/exmimo/init_exmimo2

-#!/bin/bash
-
-################################################################################
-#   OpenAirInterface
-#   Copyright(c) 1999 - 2015 Eurecom
-#
-#    OpenAirInterface is free software: you can redistribute it and/or modify
-#    it under the terms of the GNU General Public License as published by
-#    the Free Software Foundation, either version 3 of the License, or
-#    (at your option) any later version.
-#
-#
-#    OpenAirInterface is distributed in the hope that it will be useful,
-#    but WITHOUT ANY WARRANTY; without even the implied warranty of
-#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#    GNU General Public License for more details.
-#
-#    You should have received a copy of the GNU General Public License
-#    along with OpenAirInterface.The full GNU General Public License is
-#    included in this distribution in the file called "COPYING". If not,
-#    see <http://www.gnu.org/licenses/>.
-#
-#  Contact Information
-#  OpenAirInterface Admin: openair_admin@eurecom.fr
-#  OpenAirInterface Tech : openair_tech@eurecom.fr
-#  OpenAirInterface Dev  : openair4g-devel@lists.eurecom.fr
-#
-#  Address      : Eurecom, Campus SophiaTech, 450 Route des Chappes, CS 50193 - 06904 Biot Sophia Antipolis cedex, FRANCE
-#
-################################################################################
-# file init_nas_nos1
-# brief loads the nasmesh module and sets up the radio bearers (used to provide ip interface without S1 interface)
-# author Florian Kaltenberger
-#
-#######################################
-
-load_module() 
-{
-  mod_name=${1##*/}
-  mod_name=${mod_name%.*}
-  if awk "/$mod_name/ {found=1 ;exit} END {if (found!=1) exit 1}" /proc/modules
-    then
-      echo "module $mod_name already loaded: I remove it first"
-      sudo rmmod $mod_name
-  fi
-  echo loading $mod_name
-  sudo insmod $1
-}
-
-
-function main() 
-{
-PCI=`lspci -m | grep Xilinx`
-if [ -z "$PCI" ]; then
- echo "No card found. Stopping!"
- return
-fi
-
-## This part corrects the wrong configuration of the endpoint done by the bios in some machines
-echo "$PCI" | while read config_reg; do
-SLOT_NUMBER=`echo $config_reg | awk -F\" '{print $1}'`
-sudo setpci -s $SLOT_NUMBER 60.b=10
-done
-
-
-load_module $OPENAIR_DIR/targets/bin/openair_rf.ko
-sleep 1
-
-if [ ! -e /dev/openair0 ]; then 
- sudo mknod /dev/openair0 c 127 0
- sudo chmod a+rw /dev/openair0
-fi
-
-DEVICE=`echo $PCI | awk -F\" '{print $(NF-1)}' | awk '{print $2}'`
-DEVICE_SWID=${DEVICE:2:2}
-if [ $DEVICE_SWID == '0a' ]; then
-   echo "Using firware version 10"
-   $OPENAIR_DIR/targets/bin/updatefw -s 0x43fffff0 -b -f $OPENAIR_TARGETS/ARCH/EXMIMO/USERSPACE/OAI_FW_INIT/sdr_expressmimo2_v10
-else
-   echo 'No corresponding firmware found'
-   return
-fi
-}
-
-main "$@"

cmake_targets/autotests/tools/find_usb_path.bash

+#!/bin/bash
+
+#arg1 idVendor
+#arg2 idProduct
+argIdVendor=$1
+argIdProduct=$2
+
+echo $1
+echo $2
+
+for X in /sys/bus/usb/devices/*; do 
+    #echo "$X"
+    idVendor=`cat "$X/idVendor" 2>/dev/null` 
+    idProduct=`cat "$X/idProduct" 2>/dev/null`
+    if [ "$argIdVendor" == "$idVendor" ] && [ "$argIdProduct" == "$idProduct" ]
+    then
+      echo "$X"
+    fi
+done
+

cmake_targets/autotests/tools/iperf_script

+#!/bin/bash
+
+#arg1 timeout to wait before running the script
+#arg2 interface
+#arg3 iperf arguments
+
+args=($*)
+timeout=${args[0]}
+iface=${args[1]}
+iperf_args=(${args[@]:2})
+
+#array=${1:-1}
+echo "args =  ${args[@]}"
+echo "timeout = $timeout"
+echo "iface = $iface"
+echo "iperf_args = ${iperf_args[@]}"
+
+sleep $timeout
+
+while true; do var=`ifconfig $iface` ;sleep 1; if [ "$var" != "" ]; then break; fi ; done ; sleep 5
+
+iperf ${iperf_args[@]}

cmake_targets/build_oai

@@ -45,6 +45,8 @@ MSC_GEN="False"
 XFORMS="False"
 PRINT_STATS="False"
 VCD_TIMING="False"
+LOWLATENCY_FLAG_USER="False"
+FORCE_LOWLATENCY_FLAG_USER=""
 REL="Rel10"
 HW="EXMIMO"
 NOS1=0
@@ -113,7 +115,10 @@ Options
    Shows detailed compilation instructions in makefile
 --cflags_processor
    Manually Add CFLAGS of processor if they are not detected correctly by script. Only add these flags if you know your processor supports them. Example flags: -msse3 -msse4.1 -msse4.2 -mavx2
-
+--disable-deadline
+   Disables deadline scheduler of Linux kernel (>=3.14.x).
+--enable-deadline
+   Disables deadline scheduler of Linux kernel (>=3.14.x). 
 Usage (first build):
  oaisim (eNB + UE): ./build_oai -I -g --oaisim -x --install-system-files
  Eurecom EXMIMO + COTS UE : ./build_oai -I -g --eNB -x --install-system-files
@@ -225,6 +230,15 @@ function main() {
             CFLAGS_PROCESSOR_USER=$2
             echo_info "setting CPU FLAGS from USER to: $CFLAGS_PROCESSOR_USER"
             shift 2;;
+       --disable-deadline)
+            FORCE_LOWLATENCY_FLAG_USER="False"
+            echo_info "Disabling the usage of deadline scheduler"
+            shift 1;;
+       --enable-deadline)
+            FORCE_LOWLATENCY_FLAG_USER="True"
+            echo_info "Enabling the usage of deadline scheduler"
+            shift 1;;
+
         -h | --help)
             print_help
             exit 1;;
@@ -235,6 +249,31 @@ function main() {
    esac
   done
 
+  #Now we set flags to enable deadline scheduler settings
+  #By default: USRP: disable, 
+  #By default: BLADERF: enable,
+  #By default: EXMIMO: enable
+  if [ "$FORCE_LOWLATENCY_FLAG_USER" = "" ]; then
+     if [ "$HW" = "EXMIMO" ] ; then 
+        LOWLATENCY_FLAG_USER="True"
+     elif [ "$HW" = "ETHERNET" ] ; then 
+        LOWLATENCY_FLAG_USER="True"
+     elif [ "$HW" = "OAI_USRP" ] ; then 
+        LOWLATENCY_FLAG_USER="False"
+     elif [ "$HW" = "OAI_BLADERF" ] ; then 
+        LOWLATENCY_FLAG_USER="False"
+     elif [ "$HW" = "None" ] ; then 
+        LOWLATENCY_FLAG_USER="False"
+     else 
+        echo_error "Unknown HW type $HW. Exiting now..."
+        exit 
+     fi
+  else
+     LOWLATENCY_FLAG_USER=$FORCE_LOWLATENCY_FLAG_USER
+  fi
+
+  echo_info "Flags for Deadline scheduler: $LOWLATENCY_FLAG_USER"
+
   ############################################
   # setting and printing OAI envs, we should check here
   ############################################
@@ -321,6 +360,7 @@ function main() {
     echo "set ( ENABLE_VCD_FIFO $VCD_TIMING )"     >>  $cmake_file
     echo "set ( RF_BOARD \"${HW}\")"               >>  $cmake_file
     echo "set(PACKAGE_NAME \"${lte_exec}\")"     >>  $cmake_file
+    echo "set (LOWLATENCY \"${LOWLATENCY_FLAG_USER}\" )"    >>$cmake_file
     echo 'include(${CMAKE_CURRENT_SOURCE_DIR}/../CMakeLists.txt)' >> $cmake_file
     cd  $DIR/$lte_build_dir/build
     cmake ..
@@ -539,6 +579,7 @@ function main() {
     echo "set(ENABLE_ITTI False )"     					 >>  $cmake_file
     echo "set(RF_BOARD \"${HW}\")"               >>  $cmake_file
     echo 'set(PACKAGE_NAME "\"rrh_gw\"")' >>  $cmake_file
+    echo "set(LOWLATENCY \"${LOWLATENCY_FLAG_USER}\")"    >>$cmake_file
     echo 'include(${CMAKE_CURRENT_SOURCE_DIR}/../CMakeLists.txt)' >> $cmake_file
 
     [ "$CLEAN" = "1" ] && rm -rf $DIR/rrh_gw/build

cmake_targets/tools/build_helper

@@ -163,11 +163,16 @@ install_gnutls_from_source(){
 
 
 check_install_usrp_uhd_driver(){
+        #first we remove old installation
+        $SUDO apt-get remove uhd libuhd-dev libuhd003 uhd-host -y
         v=$(lsb_release -cs)
-        $SUDO apt-add-repository "deb http://files.ettus.com/binaries/uhd/repo/uhd/ubuntu/$v $v main"
+        $SUDO apt-add-repository --remove "deb http://files.ettus.com/binaries/uhd/repo/uhd/ubuntu/$v $v main"
+        #The new USRP repository
+        $SUDO add-apt-repository ppa:ettusresearch/uhd -y
         $SUDO apt-get update
         $SUDO apt-get -y install  python python-tk libboost-all-dev libusb-1.0-0-dev
-        $SUDO apt-get -y install -t `lsb_release -cs` uhd --force-yes
+        $SUDO apt-get -y install libuhd-dev libuhd003 uhd-host
+        $SUDO uhd_images_downloader 
 }
 check_install_bladerf_driver(){
 	$SUDO add-apt-repository -y ppa:bladerf/bladerf
@@ -263,7 +268,8 @@ check_install_oai_software() {
 	python-pip \
 	pydb \
 	wvdial \
-        python-numpy
+        python-numpy \
+        sshpass
     $SUDO update-alternatives --set liblapack.so /usr/lib/atlas-base/atlas/liblapack.so
     if [ `lsb_release -rs` = '12.04' ] ; then
         install_nettle_from_source

cmake_targets/tools/stop_exmimo2

+#!/bin/bash
+
+echo "This script stops exmimo2 as it might be transmitting signal by itself if lte-softmodem terminates incorrectly"
+
+if [ "$OPENAIR_DIR" == "" ]; then
+  echo "Is the OPENAIR_DIR path set correctly? Exiting now"
+  exit
+fi
+
+#load the module
+sudo -E $OPENAIR_DIR/cmake_targets/tools/init_exmimo2
+
+#now we stop the card from transmitting anything
+cd $OPENAIR_DIR/targets/bin
+sudo -E octave $OPENAIR_DIR/cmake_targets/tools/exmimo_stop_octave.m
+