CI: adding support for T tracing

  -- tracing is done w/ a single UE attach / detach
  -- traffic b/ EPC and eNB is captured by tshark
Signed-off-by: Raphael Defosseux <raphael.defosseux@eurecom.fr>

ci-scripts/active_net_interfaces.awk

+/inet.*brd/{print "interfaceToUse="$NF"done"}

ci-scripts/main.py

@@ -102,6 +102,7 @@ class SSHConnection():
 		self.iperf_args = ''
 		self.iperf_packetloss_threshold = ''
 		self.iperf_profile = ''
+		self.nbMaxUEtoAttach = -1
 		self.UEDevices = []
 		self.CatMDevices = []
 		self.UEIPAddresses = []
@@ -358,6 +359,18 @@ class SSHConnection():
 			self.CreateHtmlTestRow(self.Initialize_eNB_args, 'KO', pStatus)
 			self.CreateHtmlFooter(False)
 			sys.exit(1)
+		# If tracer options is on, running tshark on EPC side and capture traffic b/ EPC and eNB
+		result = re.search('T_stdout', str(self.Initialize_eNB_args))
+		if result is not None:
+			self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
+			self.command('ip addr show | awk -f /tmp/active_net_interfaces.awk | egrep -v "lo|tun"', '\$', 5)
+			result = re.search('interfaceToUse=(?P<eth_interface>[a-zA-Z0-9\-\_]+)done', str(self.ssh.before))
+			if result is not None:
+				eth_interface = result.group('eth_interface')
+				logging.debug('\u001B[1m Launching tshark on interface ' + eth_interface + '\u001B[0m')
+				self.command('echo ' + self.EPCPassword + ' | sudo -S rm -f /tmp/enb_' + SSH.testCase_id + '_s1log.pcap', '\$', 5)
+				self.command('echo $USER; nohup sudo tshark -f "host ' + self.eNBIPAddress +'" -i ' + eth_interface + ' -w /tmp/enb_' + SSH.testCase_id + '_s1log.pcap > /tmp/tshark.log 2>&1 &', self.EPCUserName, 5)
+			self.close()
 		self.open(self.eNBIPAddress, self.eNBUserName, self.eNBPassword)
 		self.command('cd ' + self.eNBSourceCodePath, '\$', 5)
 		# Initialize_eNB_args usually start with -O and followed by the location in repository
@@ -365,6 +378,14 @@ class SSHConnection():
 		extIdx = full_config_file.find('.conf')
 		if (extIdx > 0):
 			extra_options = full_config_file[extIdx + 5:]
+			# if tracer options is on, compiling and running T Tracer
+			result = re.search('T_stdout', str(extra_options))
+			if result is not None:
+				logging.debug('\u001B[1m Compiling and launching T Tracer\u001B[0m')
+				self.command('cd common/utils/T/tracer', '\$', 5)
+				self.command('make', '\$', 10)
+				self.command('echo $USER; nohup ./record -d ../T_messages.txt -o ' + self.eNBSourceCodePath + '/cmake_targets/enb_' + SSH.testCase_id + '_record.raw -ON -off VCD -off HEAVY -off LEGACY_GROUP_TRACE -off LEGACY_GROUP_DEBUG > ' + self.eNBSourceCodePath + '/cmake_targets/enb_' + SSH.testCase_id + '_record.log 2>&1 &', self.eNBUserName, 5)
+				self.command('cd ' + self.eNBSourceCodePath, '\$', 5)
 			full_config_file = full_config_file[:extIdx + 5]
 			config_path, config_file = os.path.split(full_config_file)
 		else:
@@ -608,11 +629,14 @@ class SSHConnection():
 		multi_jobs = []
 		status_queue = SimpleQueue()
 		lock = Lock()
+		nb_ue_to_connect = 0
 		for device_id in self.UEDevices:
+			if (self.nbMaxUEtoAttach == -1) or (nb_ue_to_connect < self.nbMaxUEtoAttach):
 				p = Process(target = SSH.AttachUE_common, args = (device_id, status_queue, lock,))
 				p.daemon = True
 				p.start()
 				multi_jobs.append(p)
+			nb_ue_to_connect = nb_ue_to_connect + 1
 		for job in multi_jobs:
 			job.join()
 
@@ -1645,8 +1669,21 @@ class SSHConnection():
 			self.command('echo ' + self.eNBPassword + ' | sudo -S killall --signal SIGKILL lte-softmodem || true', '\$', 5)
 		self.close()
 		result = re.search('enb_', str(self.eNBLogFile))
+		# If tracer options is on, stopping tshark on EPC side
+		result = re.search('T_stdout', str(self.Initialize_eNB_args))
+		if result is not None:
+			self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
+			logging.debug('\u001B[1m Stopping tshark \u001B[0m')
+			self.command('echo ' + self.EPCPassword + ' | sudo -S killall --signal SIGKILL tshark', '\$', 5)
+			time.sleep(1)
+			pcap_log_file = self.eNBLogFile.replace('.log', '_s1log.pcap')
+			self.command('echo ' + self.EPCPassword + ' | sudo -S chmod 666 /tmp/' + pcap_log_file, '\$', 5)
+			self.copyin(self.EPCIPAddress, self.EPCUserName, self.EPCPassword, '/tmp/' + pcap_log_file, '.')
+			self.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, pcap_log_file, self.eNBSourceCodePath + '/cmake_targets/.')
+			self.close()
 		if result is not None:
 			self.copyin(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, self.eNBSourceCodePath + '/cmake_targets/' + self.eNBLogFile, '.')
+			logging.debug('\u001B[1m Analyzing eNB logfile \u001B[0m')
 			logStatus = self.AnalyzeLogFile_eNB()
 			if (logStatus < 0):
 				self.CreateHtmlTestRow('N/A', 'KO', logStatus)
@@ -1672,11 +1709,8 @@ class SSHConnection():
 			self.command('cd scripts', '\$', 5)
 			self.command('rm -f ./kill_hss.sh', '\$', 5)
 			self.command('echo ' + self.EPCPassword + ' | sudo -S daemon --name=simulated_hss --stop', '\$', 5)
-			time.sleep(2)
-			self.command('ps -aux | egrep --color=never "hss_sim|simulated_hss" | grep -v grep | awk \'BEGIN{n=0}{pidId[n]=$2;n=n+1}END{print "kill -9 " pidId[0] " " pidId[1]}\' > ./kill_hss.sh', '\$', 5)
-			self.command('chmod 755 ./kill_hss.sh', '\$', 5)
-			self.command('echo ' + self.EPCPassword + ' | sudo -S ./kill_hss.sh', '\$', 5)
-			self.command('rm ./kill_hss.sh', '\$', 5)
+			time.sleep(1)
+			self.command('echo ' + self.EPCPassword + ' | sudo -S killall --signal SIGKILL hss_sim', '\$', 5)
 		self.close()
 		self.CreateHtmlTestRow('N/A', 'OK', ALL_PROCESSES_OK)
 
@@ -1763,8 +1797,8 @@ class SSHConnection():
 		self.command('cd ' + self.eNBSourceCodePath, '\$', 5)
 		self.command('cd cmake_targets', '\$', 5)
 		self.command('echo ' + self.eNBPassword + ' | sudo -S rm -f enb.log.zip', '\$', 5)
-		self.command('echo ' + self.eNBPassword + ' | sudo -S zip enb.log.zip enb*.log core*', '\$', 60)
-		self.command('echo ' + self.eNBPassword + ' | sudo -S rm enb*.log core*', '\$', 5)
+		self.command('echo ' + self.eNBPassword + ' | sudo -S zip enb.log.zip enb*.log core* enb_*record.raw enb_*.pcap', '\$', 60)
+		self.command('echo ' + self.eNBPassword + ' | sudo -S rm enb*.log core* enb_*record.raw enb_*.pcap', '\$', 5)
 		self.close()
 
 	def LogCollectPing(self):
@@ -2071,6 +2105,13 @@ def GetParametersFromXML(action):
 		if (SSH.eNB_instance is None):
 			SSH.eNB_instance = '0'
 
+	if action == 'Attach_UE':
+		nbMaxUEtoAttach = test.findtext('nbMaxUEtoAttach')
+		if (nbMaxUEtoAttach is None):
+			SSH.nbMaxUEtoAttach = -1
+		else:
+			SSH.nbMaxUEtoAttach = int(nbMaxUEtoAttach)
+
 	if action == 'Ping':
 		SSH.ping_args = test.findtext('ping_args')
 		SSH.ping_packetloss_threshold = test.findtext('ping_packetloss_threshold')
@@ -2248,6 +2289,7 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE):
 		sys.exit('Insufficient Parameter')
 
 	SSH.copyout(SSH.EPCIPAddress, SSH.EPCUserName, SSH.EPCPassword, sys.path[0] + "/tcp_iperf_stats.awk", "/tmp")
+	SSH.copyout(SSH.EPCIPAddress, SSH.EPCUserName, SSH.EPCPassword, sys.path[0] + "/active_net_interfaces.awk", "/tmp")
 	SSH.CreateHtmlHeader()
 
 	#read test_case_list.xml file

ci-scripts/xml_files/enb_usrp210_band7.xml

@@ -24,6 +24,7 @@
 	<TestCaseRequestedList>
  010101
  050101 060101 070101 040101
+ 030100 040300 040401 040201 030201
  030101 040301 040501 040603 040604 040605 040606 040607 040641 040642 040643 040644 040401 040201 030201
  030111 040301 040511 040613 040614 040615 040616 040617 040651 040652 040653 040654 040401 040201 030201
  030121 040301 040521 040623 040624 040625 040626 040627 040662 040661 040663 040664 040401 040201 030201
@@ -37,6 +38,12 @@
 		<Build_eNB_args>-w USRP -c --eNB</Build_eNB_args>
 	</testCase>
 
+	<testCase id="030100">
+		<class>Initialize_eNB</class>
+		<desc>Initialize eNB (FDD/Band7/5MHz) with T Tracer on</desc>
+		<Initialize_eNB_args>-O ci-scripts/conf_files/enb.band7.tm1.25PRB.usrpb210.conf --T_stdout 0</Initialize_eNB_args>
+	</testCase>
+
 	<testCase id="030101">
 		<class>Initialize_eNB</class>
 		<desc>Initialize eNB (FDD/Band7/5MHz)</desc>
@@ -70,6 +77,12 @@
 		<desc>Terminate UE</desc>
 	</testCase>
 
+        <testCase id="040300">
+                <class>Attach_UE</class>
+                <desc>Attach single UE</desc>
+                <nbMaxUEtoAttach>1</nbMaxUEtoAttach>
+        </testCase>
+
         <testCase id="040301">
                 <class>Attach_UE</class>
                 <desc>Attach UE</desc>

ci-scripts/xml_files/enb_usrpb210_band40.xml

@@ -24,6 +24,7 @@
         <TestCaseRequestedList>
  010101
  050101 060101 070101 040101
+ 030100 040300 040401 040201 030201
  030104 040301 040501 040602 040601 040603 040642 040641 040643 040401 040201 030201
  030114 040301 040511 040612 040611 040613 040652 040651 040653 040401 040201 030201
  050201 060201 070201
@@ -40,6 +41,12 @@
 		<Build_eNB_args>-w USRP -c --eNB</Build_eNB_args>
 	</testCase>
 
+	<testCase id="030100">
+		<class>Initialize_eNB</class>
+		<desc>Initialize eNB (TDD/Band40/5MHz) with T Tracer on</desc>
+		<Initialize_eNB_args>-O ci-scripts/conf_files/enb.band40.tm1.25PRB.FairScheduler.usrpb210.conf --T_stdout 0</Initialize_eNB_args>
+	</testCase>
+
 	<testCase id="030104">
 		<class>Initialize_eNB</class>
 		<desc>Initialize eNB (TDD/Band40/5MHz)</desc>
@@ -73,6 +80,12 @@
 		<desc>Terminate UE</desc>
 	</testCase>
 
+        <testCase id="040300">
+                <class>Attach_UE</class>
+                <desc>Attach single UE</desc>
+                <nbMaxUEtoAttach>1</nbMaxUEtoAttach>
+        </testCase>
+
         <testCase id="040301">
                 <class>Attach_UE</class>
                 <desc>Attach UE</desc>