CI: better handling of test failures

  -- IperfNoS1 when eNB crashed
  -- LTE-M module failed attachment
  -- LTE-M module pinging
Signed-off-by: Raphael Defosseux <raphael.defosseux@eurecom.fr>

ci-scripts/main.py

@@ -521,6 +521,35 @@ class SSHConnection():
 		# here add a check if git clone or git fetch went smoothly
 		self.command('git config user.email "jenkins@openairinterface.org"', '\$', 5)
 		self.command('git config user.name "OAI Jenkins"', '\$', 5)
+		self.command('ls *.txt', '\$', 5)
+		result = re.search('LAST_BUILD_INFO', str(self.ssh.before))
+		if result is not None:
+			mismatch = False
+			self.command('grep SRC_COMMIT LAST_BUILD_INFO.txt', '\$', 2)
+			result = re.search(self.ranCommitID, str(self.ssh.before))
+			if result is None:
+				mismatch = True
+			self.command('grep MERGED_W_TGT_BRANCH LAST_BUILD_INFO.txt', '\$', 2)
+			if (self.ranAllowMerge):
+				result = re.search('YES', str(self.ssh.before))
+				if result is None:
+					mismatch = True
+				self.command('grep TGT_BRANCH LAST_BUILD_INFO.txt', '\$', 2)
+				if self.ranTargetBranch == '':
+					result = re.search('develop', str(self.ssh.before))
+				else:
+					result = re.search(self.ranTargetBranch, str(self.ssh.before))
+				if result is None:
+					mismatch = True
+			else:
+				result = re.search('NO', str(self.ssh.before))
+				if result is None:
+					mismatch = True
+			if not mismatch:
+				self.close()
+				self.CreateHtmlTestRow(self.Build_eNB_args, 'OK', ALL_PROCESSES_OK)
+				return
+
 		self.command('echo ' + self.UEPassword + ' | sudo -S git clean -x -d -ff', '\$', 30)
 		# if the commit ID is provided use it to point to it
 		if self.ranCommitID != '':
@@ -549,10 +578,22 @@ class SSHConnection():
 		self.command('mkdir -p build_log_' + self.testCase_id, '\$', 5)
 		self.command('mv log/* ' + 'build_log_' + self.testCase_id, '\$', 5)
 		self.command('mv compile_oai_ue.log ' + 'build_log_' + self.testCase_id, '\$', 5)
-		self.close()
 		if buildStatus:
+			# Generating a BUILD INFO file
+			self.command('echo "SRC_BRANCH: ' + self.ranBranch + '" > ../LAST_BUILD_INFO.txt', '\$', 2)
+			self.command('echo "SRC_COMMIT: ' + self.ranCommitID + '" >> ../LAST_BUILD_INFO.txt', '\$', 2)
+			if (self.ranAllowMerge):
+				self.command('echo "MERGED_W_TGT_BRANCH: YES" >> ../LAST_BUILD_INFO.txt', '\$', 2)
+				if self.ranTargetBranch == '':
+					self.command('echo "TGT_BRANCH: develop" >> ../LAST_BUILD_INFO.txt', '\$', 2)
+				else:
+					self.command('echo "TGT_BRANCH: ' + self.ranTargetBranch + '" >> ../LAST_BUILD_INFO.txt', '\$', 2)
+			else:
+				self.command('echo "MERGED_W_TGT_BRANCH: NO" >> ../LAST_BUILD_INFO.txt', '\$', 2)
+			self.close()
 			self.CreateHtmlTestRow(self.Build_OAI_UE_args, 'OK', ALL_PROCESSES_OK, 'OAI UE')
 		else:
+			self.close()
 			logging.error('\u001B[1m Building OAI UE Failed\u001B[0m')
 			self.CreateHtmlTestRow(self.Build_OAI_UE_args, 'KO', ALL_PROCESSES_OK, 'OAI UE')
 			self.CreateHtmlTabFooter(False)
@@ -1081,7 +1122,7 @@ class SSHConnection():
 		html_queue = SimpleQueue()
 		self.checkDevTTYisUnlocked()
 		if attach_status:
-			html_cell = '<pre style="background-color:white">CAT-M module\nAttachment Completed in ' + str(attach_cnt+4) + ' seconds'
+			html_cell = '<pre style="background-color:white">CAT-M module Attachment Completed in ' + str(attach_cnt+4) + ' seconds'
 			if (nRSRQ is not None) and (nRSRP is not None):
 				html_cell += '\n   RSRQ = ' + str(-20+(nRSRQ/2)) + ' dB'
 				html_cell += '\n   RSRP = ' + str(-140+nRSRP) + ' dBm</pre>'
@@ -1090,9 +1131,11 @@ class SSHConnection():
 			html_queue.put(html_cell)
 			self.CreateHtmlTestRowQueue('N/A', 'OK', 1, html_queue)
 		else:
-			html_cell = '<pre style="background-color:white">CAT-M module\nAttachment Failed</pre>'
+			logging.error('\u001B[1m CAT-M module Attachment Failed\u001B[0m')
+			html_cell = '<pre style="background-color:white">CAT-M module Attachment Failed</pre>'
 			html_queue.put(html_cell)
 			self.CreateHtmlTestRowQueue('N/A', 'KO', 1, html_queue)
+			self.AutoTerminateUEandeNB()
 
 	def PingCatM(self):
 		if self.EPCIPAddress == '' or self.EPCUserName == '' or self.EPCPassword == '' or self.EPCSourceCodePath == '':
@@ -1103,7 +1146,7 @@ class SSHConnection():
 		pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE)
 		if (pStatus < 0):
 			self.CreateHtmlTestRow(self.ping_args, 'KO', pStatus)
-			self.prematureExit = True
+			self.AutoTerminateUEandeNB()
 			return
 		try:
 			statusQueue = SimpleQueue()
@@ -1122,6 +1165,8 @@ class SSHConnection():
 				if result is not None:
 					moduleIPAddr = result.group('ipaddr')
 				else:
+					self.CreateHtmlTestRow(self.ping_args, 'KO', pStatus)
+					self.AutoTerminateUEandeNB()
 					return
 			ping_time = re.findall("-c (\d+)",str(self.ping_args))
 			device_id = 'catm'
@@ -2353,15 +2398,16 @@ class SSHConnection():
 			os.kill(os.getppid(),signal.SIGUSR1)
 
 	def IperfNoS1(self):
+		if self.eNBIPAddress == '' or self.eNBUserName == '' or self.eNBPassword == '' or self.UEIPAddress == '' or self.UEUserName == '' or self.UEPassword == '':
+			Usage()
+			sys.exit('Insufficient Parameter')
 		check_eNB = True
 		check_OAI_UE = True
 		pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE)
 		if (pStatus < 0):
 			self.CreateHtmlTestRow(self.iperf_args, 'KO', pStatus)
 			self.AutoTerminateUEandeNB()
-		if self.eNBIPAddress == '' or self.eNBUserName == '' or self.eNBPassword == '' or self.UEIPAddress == '' or self.UEUserName == '' or self.UEPassword == '':
-			Usage()
-			sys.exit('Insufficient Parameter')
+			return
 		server_on_enb = re.search('-R', str(self.iperf_args))
 		if server_on_enb is not None:
 			iServerIPAddr = self.eNBIPAddress
@@ -4399,7 +4445,11 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re
 	while cnt < SSH.repeatCounts[0] and SSH.prematureExit:
 		SSH.prematureExit = False
 		for test_case_id in todo_tests:
+			if SSH.prematureExit:
+				break
 			for test in all_tests:
+				if SSH.prematureExit:
+					break
 				id = test.get('id')
 				if test_case_id != id:
 					continue
@@ -4480,10 +4530,6 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re
 					SSH.Perform_X2_Handover()
 				else:
 					sys.exit('Invalid action')
-				if SSH.prematureExit:
-					break
-			if SSH.prematureExit:
-				break
 		cnt += 1
 	if cnt == SSH.repeatCounts[0] and SSH.prematureExit:
 		logging.debug('Testsuite failed ' + str(cnt) + ' time(s)')

ci-scripts/xml_files/enb_ue_usrp210_band7_epc_start.xml

@@ -25,10 +25,23 @@
 	<htmlTabName>EPC-Start</htmlTabName>
 	<htmlTabIcon>log-in</htmlTabIcon>
 	<TestCaseRequestedList>
+ 010101 090101
  050101 060101 070101
 	</TestCaseRequestedList>
 	<TestCaseExclusionList></TestCaseExclusionList>
 
+        <testCase id="010101">
+                <class>Build_eNB</class>
+                <desc>Build eNB (USRP)</desc>
+                <Build_eNB_args>-w USRP -c --eNB</Build_eNB_args>
+        </testCase>
+
+        <testCase id="090101">
+                <class>Build_OAI_UE</class>
+                <desc>Build OAI UE</desc>
+                <Build_OAI_UE_args>-w USRP --UE</Build_OAI_UE_args>
+        </testCase>
+
 	<testCase id="050101">
 		<class>Initialize_HSS</class>
 		<desc>Initialize HSS</desc>

ci-scripts/xml_files/enb_ue_usrp210_band7_test_05mhz_tm1.xml

@@ -24,7 +24,7 @@
 	<htmlTabRef>test-05-tm1-nos1-tunnel</htmlTabRef>
 	<htmlTabName>Test-05MHz-TM1-noS1-tunnel</htmlTabName>
 	<htmlTabIcon>tasks</htmlTabIcon>
-	<repeatCount>2</repeatCount>
+	<repeatCount>4</repeatCount>
 	<TestCaseRequestedList>
  030201 090109
  030101 000001 090101 000002 040501 040502 000001 040601 040602 040641 040642 000001 090109 030201

ci-scripts/xml_files/enb_usrp210_band13_epc_closure.xml

@@ -25,10 +25,16 @@
 	<htmlTabName>EPC-Closure</htmlTabName>
 	<htmlTabIcon>log-out</htmlTabIcon>
 	<TestCaseRequestedList>
+ 040202
  050201 060201 070201
 	</TestCaseRequestedList>
 	<TestCaseExclusionList></TestCaseExclusionList>
 
+	<testCase id="040202">
+		<class>Terminate_CatM_module</class>
+		<desc>Terminate CAT-M Module</desc>
+	</testCase>
+
 	<testCase id="050201">
 		<class>Terminate_HSS</class>
 		<desc>Terminate HSS</desc>