CI: scenarios that are known as unstable won't be seen as failures

Signed-off-by: Raphael Defosseux <raphael.defosseux@eurecom.fr>

ci-scripts/cls_oaicitest.py

@@ -72,6 +72,9 @@ class OaiCiTest():
 		self.ADBCentralized = True
 		self.testCase_id = ''
 		self.testXMLfiles = []
+		self.testUnstable = False
+		self.testMinStableId = '999999'
+		self.testStabilityPointReached = False
 		self.desc = ''
 		self.ping_args = ''
 		self.ping_packetloss_threshold = ''
@@ -211,7 +214,7 @@ class OaiCiTest():
 			logging.error('\u001B[1m Building OAI UE Failed\u001B[0m')
 			HTML.CreateHtmlTestRow(self.Build_OAI_UE_args, 'KO', CONST.ALL_PROCESSES_OK, 'OAI UE')
 			HTML.CreateHtmlTabFooter(False)
-			sys.exit(1)
+			self.ConditionalExit()
 
 	def CheckFlexranCtrlInstallation(self,RAN,EPC):
 		if EPC.IPAddress == '' or EPC.UserName == '' or EPC.Password == '':
@@ -335,7 +338,7 @@ class OaiCiTest():
 				if (pStatus < 0):
 					HTML.CreateHtmlTestRow(self.air_interface + ' ' + self.Initialize_OAI_UE_args, 'KO', pStatus)
 					HTML.CreateHtmlTabFooter(False)
-					sys.exit(1)
+					self.ConditionalExit()
 			UE_prefix = ''
 		else:
 			UE_prefix = 'NR '
@@ -719,7 +722,7 @@ class OaiCiTest():
 				logging.debug('Using the OAI EPC HSS: not implemented yet')
 				HTML.CreateHtmlTestRow(self.ping_args, 'KO', pStatus)
 				HTML.CreateHtmlTabFooter(False)
-				sys.exit(1)
+				self.ConditionalExit()
 			else:
 				SSH.command('egrep --color=never "Allocated ipv4 addr" /opt/ltebox/var/log/xGwLog.0', '\$', 5)
 				result = re.search('Allocated ipv4 addr: (?P<ipaddr>[0-9\.]+) from Pool', SSH.getBefore())
@@ -1046,7 +1049,7 @@ class OaiCiTest():
 		if (pStatus < 0):
 			HTML.CreateHtmlTestRow('N/A', 'KO', pStatus)
 			HTML.CreateHtmlTabFooter(False)
-			sys.exit(1)
+			self.ConditionalExit()
 		multi_jobs = []
 		for device_id in self.UEDevices:
 			p = Process(target = self.RebootUE_common, args = (device_id,))
@@ -1248,7 +1251,7 @@ class OaiCiTest():
 		if (pStatus < 0):
 			HTML.CreateHtmlTestRow('N/A', 'KO', pStatus)
 			HTML.CreateHtmlTabFooter(False)
-			sys.exit(1)
+			self.ConditionalExit()
 		multi_jobs = []
 		lock = Lock()
 		status_queue = SimpleQueue()
@@ -3169,6 +3172,12 @@ class OaiCiTest():
 			HTML.CpuMHz[idx]=CpuMHz
 		SSH.close()
 
+	def ConditionalExit(self):
+		if self.testUnstable:
+			if self.testStabilityPointReached or self.testMinStableId == '999999':
+				sys.exit(0)
+		sys.exit(1)
+
 	def ShowTestID(self):
 		logging.debug('\u001B[1m----------------------------------------\u001B[0m')
 		logging.debug('\u001B[1mTest ID:' + self.testCase_id + '\u001B[0m')

ci-scripts/html.py

@@ -64,6 +64,10 @@ class HTMLManagement():
 		self.htmlTabIcons = []
 		self.testXMLfiles = []
 
+		self.testUnstable = False
+		self.testMinStableId = '999999'
+		self.testStabilityPointReached = False
+
 		self.htmleNBFailureMsg = ''
 		self.htmlUEFailureMsg = ''
 
@@ -230,13 +234,20 @@ class HTMLManagement():
 
 	def CreateHtmlTabFooter(self, passStatus):
 		if ((not self.htmlFooterCreated) and (self.htmlHeaderCreated)):
+			testOkEvenIfUnstable = False
+			if self.testUnstable and not passStatus:
+				if self.testStabilityPointReached or self.testMinStableId == '999999':
+					testOkEvenIfUnstable = True
 			self.htmlFile = open('test_results.html', 'a')
 			self.htmlFile.write('      <tr>\n')
 			self.htmlFile.write('        <th bgcolor = "#33CCFF" colspan=3>Final Tab Status</th>\n')
 			if passStatus:
 				self.htmlFile.write('        <th bgcolor = "green" colspan=' + str(2 + self.htmlUEConnected) + '><font color="white">PASS <span class="glyphicon glyphicon-ok"></span> </font></th>\n')
 			else:
-				self.htmlFile.write('        <th bgcolor = "red" colspan=' + str(2 + self.htmlUEConnected) + '><font color="white">FAIL <span class="glyphicon glyphicon-remove"></span> </font></th>\n')
+				if testOkEvenIfUnstable:
+					self.htmlFile.write('        <th bgcolor = "orange" colspan=' + str(2 + self.htmlUEConnected) + '><font color="white">KNOWN UNSTABLE SCENARIO <span class="glyphicon glyphicon-exclamation-sign"></span> </font></th>\n')
+				else:
+					self.htmlFile.write('        <th bgcolor = "red" colspan=' + str(2 + self.htmlUEConnected) + '><font color="white">FAIL <span class="glyphicon glyphicon-remove"></span> </font></th>\n')
 			self.htmlFile.write('      </tr>\n')
 			self.htmlFile.write('  </table>\n')
 			self.htmlFile.write('  </div>\n')
@@ -246,7 +257,10 @@ class HTMLManagement():
 				cmd = "sed -i -e 's/__STATE_" + self.htmlTabNames[0] + "__//' test_results.html"
 				subprocess.run(cmd, shell=True)
 			else:
-				cmd = "sed -i -e 's/__STATE_" + self.htmlTabNames[0] + "__/<span class=\"glyphicon glyphicon-remove\"><\/span>/' test_results.html"
+				if testOkEvenIfUnstable:
+					cmd = "sed -i -e 's/__STATE_" + self.htmlTabNames[0] + "__/<span class=\"glyphicon glyphicon-exclamation-sign\"><\/span>/' test_results.html"
+				else:
+					cmd = "sed -i -e 's/__STATE_" + self.htmlTabNames[0] + "__/<span class=\"glyphicon glyphicon-remove\"><\/span>/' test_results.html"
 				subprocess.run(cmd, shell=True)
 		self.htmlFooterCreated = False
 

ci-scripts/main.py

@@ -494,6 +494,9 @@ elif re.match('^FinalizeHtml$', mode, re.IGNORECASE):
 	CiTestObj.RetrieveSystemVersion('UE',HTML,RAN)
 	HTML.CreateHtmlFooter(CiTestObj.finalStatus)
 elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re.IGNORECASE):
+	logging.debug('\u001B[1m----------------------------------------\u001B[0m')
+	logging.debug('\u001B[1m  Starting Scenario \u001B[0m')
+	logging.debug('\u001B[1m----------------------------------------\u001B[0m')
 	if re.match('^TesteNB$', mode, re.IGNORECASE):
 		if RAN.eNBIPAddress == '' or RAN.ranRepository == '' or RAN.ranBranch == '' or RAN.eNBUserName == '' or RAN.eNBPassword == '' or RAN.eNBSourceCodePath == '' or EPC.IPAddress == '' or EPC.UserName == '' or EPC.Password == '' or EPC.Type == '' or EPC.SourceCodePath == '' or CiTestObj.ADBIPAddress == '' or CiTestObj.ADBUserName == '' or CiTestObj.ADBPassword == '':
 			HELP.GenericHelp(CONST.Version)
@@ -529,7 +532,14 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re
 		HTML.htmlTabRefs.append(xmlRoot.findtext('htmlTabRef',default='test-tab-0'))
 		HTML.htmlTabNames.append(xmlRoot.findtext('htmlTabName',default='Test-0'))
 		repeatCount = xmlRoot.findtext('repeatCount',default='1')
+		testStability = xmlRoot.findtext('TestUnstable',default='False')
 		CiTestObj.repeatCounts.append(int(repeatCount))
+		if testStability == 'True':
+			CiTestObj.testUnstable = True
+			HTML.testUnstable = True
+			CiTestObj.testMinStableId = xmlRoot.findtext('TestMinId',default='999999')
+			HTML.testMinStableId = CiTestObj.testMinStableId
+			logging.debug('Test is tagged as Unstable -- starting from TestID ' + str(CiTestObj.testMinStableId))
 	all_tests=xmlRoot.findall('testCase')
 
 	exclusion_tests=exclusion_tests.split()
@@ -686,11 +696,19 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re
 					HTML=ldpc.Run_PhySim(HTML,CONST,id)
 				else:
 					sys.exit('Invalid class (action) from xml')
+				if not RAN.prematureExit:
+					if CiTestObj.testCase_id == CiTestObj.testMinStableId:
+						logging.debug('Scenario has reached minimal stability point')
+						CiTestObj.testStabilityPointReached = True
+						HTML.testStabilityPointReached = True
 		CiTestObj.FailReportCnt += 1
 	if CiTestObj.FailReportCnt == CiTestObj.repeatCounts[0] and RAN.prematureExit:
 		logging.debug('Testsuite failed ' + str(CiTestObj.FailReportCnt) + ' time(s)')
 		HTML.CreateHtmlTabFooter(False)
-		sys.exit('Failed Scenario')
+		if CiTestObj.testUnstable and (CiTestObj.testStabilityPointReached or CiTestObj.testMinStableId == '999999'):
+			logging.debug('Scenario has reached minimal stability point -- Not a Failure')
+		else:
+			sys.exit('Failed Scenario')
 	else:
 		logging.info('Testsuite passed after ' + str(CiTestObj.FailReportCnt) + ' time(s)')
 		HTML.CreateHtmlTabFooter(True)

ci-scripts/xml_files/enb_usrp210_band13_test_10mhz_tm1.xml

@@ -25,6 +25,7 @@
 	<htmlTabName>Test-LTE-M-10MHz-TM1</htmlTabName>
 	<htmlTabIcon>tasks</htmlTabIcon>
 	<repeatCount>1</repeatCount>
+	<TestUnstable>True</TestUnstable>
 	<TestCaseRequestedList>
  030201
  040102

ci-scripts/xml_files/enb_usrp210_band40_test_05mhz_tm2.xml

@@ -26,6 +26,8 @@
 	<htmlTabName>Test-05MHz-TM2</htmlTabName>
 	<htmlTabIcon>tasks</htmlTabIcon>
 	<repeatCount>2</repeatCount>
+	<TestUnstable>True</TestUnstable>
+	<TestMinId>040608</TestMinId>
         <TestCaseRequestedList>
  030201
  040101

ci-scripts/xml_files/enb_usrp210_band40_test_10mhz_tm1.xml

@@ -24,6 +24,8 @@
 	<htmlTabRef>test-10-tm1</htmlTabRef>
 	<htmlTabName>Test-10MHz-TM1</htmlTabName>
 	<htmlTabIcon>tasks</htmlTabIcon>
+	<TestUnstable>True</TestUnstable>
+	<TestMinId>040615</TestMinId>
 	<TestCaseRequestedList>
  030201
  040101

ci-scripts/xml_files/enb_usrp210_band40_test_20mhz_tm1.xml

@@ -25,6 +25,8 @@
 	<htmlTabName>Test-20MHz-TM1</htmlTabName>
 	<htmlTabIcon>tasks</htmlTabIcon>
 	<repeatCount>4</repeatCount>
+	<TestUnstable>True</TestUnstable>
+	<TestMinId>040623</TestMinId>
 	<TestCaseRequestedList>
  030201
  040101

ci-scripts/xml_files/enb_usrp210_band40_test_20mhz_tm1_default_scheduler.xml

@@ -24,6 +24,8 @@
 	<htmlTabRef>test-20-tm1-defSched</htmlTabRef>
 	<htmlTabName>Test-20MHz-TM1-default-scheduler</htmlTabName>
 	<htmlTabIcon>tasks</htmlTabIcon>
+	<TestUnstable>True</TestUnstable>
+	<TestMinId>040522</TestMinId>
 	<TestCaseRequestedList>
  030201
  040101

ci-scripts/xml_files/if4p5_usrp210_band40_test_10mhz.xml

@@ -25,6 +25,8 @@
 	<htmlTabName>Test-10MHz</htmlTabName>
 	<htmlTabIcon>tasks</htmlTabIcon>
 	<repeatCount>2</repeatCount>
+	<TestUnstable>True</TestUnstable>
+	<TestMinId>040612</TestMinId>
 	<TestCaseRequestedList>
  030201
  040101

ci-scripts/xml_files/x2ho_enb_usrp210_band13_test_05mhz_tm1.xml

@@ -26,6 +26,7 @@
 	<htmlTabName>Test-X2-HO-FDD-05MHz-TM1</htmlTabName>
 	<htmlTabIcon>tasks</htmlTabIcon>
 	<repeatCount>2</repeatCount>
+	<TestUnstable>True</TestUnstable>
 	<TestCaseRequestedList>
  030201 030202
  040101