Add retx checkers to make CI test fail on retx thresholds

ci-scripts/cls_containerize.py

@@ -106,6 +106,9 @@ class Containerize():
 		self.registrySvrId = ''
 		self.testSvrId = ''
 
+		#checkers from xml
+		self.ran_checkers={}
+
 #-----------------------------------------------------------
 # Container management functions
 #-----------------------------------------------------------
@@ -624,7 +627,7 @@ class Containerize():
 		else:
 			if containerToKill:
 				logging.debug('\u001B[1m Analyzing ' + nodeB_prefix + 'NB logfile \u001B[0m ' + self.eNB_logFile[self.eNB_instance])
-				logStatus = RAN.AnalyzeLogFile_eNB(self.eNB_logFile[self.eNB_instance], HTML)
+				logStatus = RAN.AnalyzeLogFile_eNB(self.eNB_logFile[self.eNB_instance], HTML, self.ran_checkers)
 			else:
 				logStatus = 0
 			if (logStatus < 0):
@@ -788,9 +791,10 @@ class Containerize():
 					continue
 
 				logging.debug('\u001B[1m Analyzing xNB logfile ' + filename + ' \u001B[0m')
-				logStatus = RAN.AnalyzeLogFile_eNB(filename, HTML)
+				logStatus = RAN.AnalyzeLogFile_eNB(filename, HTML, self.ran_checkers)
 				if (logStatus < 0):
 					fullStatus = False
+					self.exitStatus = 1
 					HTML.CreateHtmlTestRow(RAN.runtime_stats, 'KO', logStatus)
 				else:
 					HTML.CreateHtmlTestRow(RAN.runtime_stats, 'OK', CONST.ALL_PROCESSES_OK)

ci-scripts/constants.py

@@ -45,6 +45,7 @@ ENB_PROCESS_REALTIME_ISSUE = -13
 ENB_PROCESS_NOLOGFILE_TO_ANALYZE = -14
 ENB_PROCESS_SLAVE_RRU_NOT_SYNCED = -15
 ENB_REAL_TIME_PROCESSING_ISSUE = -16
+ENB_RETX_ISSUE = -17
 HSS_PROCESS_FAILED = -2
 HSS_PROCESS_OK = +2
 MME_PROCESS_FAILED = -3

ci-scripts/main.py

@@ -401,6 +401,12 @@ def GetParametersFromXML(action):
 		string_field=test.findtext('nb_healthy')
 		if (string_field is not None):
 			CONTAINERS.nb_healthy[0] = int(string_field)
+		string_field=test.findtext('d_retx_th')
+		if (string_field is not None):
+			CONTAINERS.ran_checkers['d_retx_th']= string_field
+		string_field=test.findtext('u_retx_th')
+		if (string_field is not None):
+			CONTAINERS.ran_checkers['u_retx_th']= string_field
 
 	elif action == 'PingFromContainer':
 		string_field = test.findtext('container_name')

ci-scripts/ran.py

@@ -39,6 +39,7 @@ import time
 from multiprocessing import Process, Lock, SimpleQueue
 import yaml
 
+
 #-----------------------------------------------------------
 # OAI Testing modules
 #-----------------------------------------------------------
@@ -758,7 +759,7 @@ class RANManagement():
 		mySSH.command('echo ' + self.eNBPassword + ' | sudo -S rm enb*.log core* enb_*record.raw enb_*.pcap gnb_*.pcap enb_*txt physim_*.log *stats.log *monitor.pickle *monitor*.png ping*.log.png log/*/*.log log/*/*.pcap', '\$', 15)
 		mySSH.close()
 
-	def AnalyzeLogFile_eNB(self, eNBlogFile, HTML):
+	def AnalyzeLogFile_eNB(self, eNBlogFile, HTML, checkers={}):
 		if (not os.path.isfile('./' + eNBlogFile)):
 			return -1
 		enb_log_file = open('./' + eNBlogFile, 'r')
@@ -819,6 +820,8 @@ class RANManagement():
 		gnb_markers ={'SgNBReleaseRequestAcknowledge': [],'FAILURE': [], 'scgFailureInformationNR-r15': [], 'SgNBReleaseRequest': [], 'Detected UL Failure on PUSCH':[]}
 		nodeB_prefix_found = False
 		RealTimeProcessingIssue = False
+		DLRetxIssue = False
+		ULRetxIssue = False
 	
 		line_cnt=0 #log file line counter
 		for line in enb_log_file.readlines():
@@ -1117,14 +1120,56 @@ class RANManagement():
 			logging.debug(statMsg)
 			htmleNBFailureMsg += htmlMsg
 
-			#ulsch and dlsch statistics
+			#ulsch and dlsch statistics and checkers
+			#print statistics into html
 			if len(dlsch_ulsch_stats)!=0: #check if dictionary is not empty
+				#for each dictionary key, generate the msg for html as information
 				statMsg=''
-				for key in dlsch_ulsch_stats: #for each dictionary key
+				for key in dlsch_ulsch_stats: 
 					statMsg += dlsch_ulsch_stats[key] + '\n' 
 					logging.debug(dlsch_ulsch_stats[key])
 				htmleNBFailureMsg += statMsg
 
+			#checker
+			if (len(dlsch_ulsch_stats)!=0) and (len(checkers)!=0):
+				if 'd_retx_th' in checkers:
+					checkers['d_retx_th'] = [float(x) for x in checkers['d_retx_th'].split(',')]
+					dlsch_checker_status = list(0 for i in checkers['d_retx_th'])#status 0 / -1
+					d_perc_retx = list(0 for i in checkers['d_retx_th'])#results in %
+
+				if 'u_retx_th' in checkers:
+					checkers['u_retx_th'] = [float(x) for x in checkers['u_retx_th'].split(',')]
+					ulsch_checker_status = list(0 for i in checkers['u_retx_th'])
+					u_perc_retx = list(0 for i in checkers['u_retx_th'])
+
+				#ul and dl retransmissions checkers
+				#NOTICE:  DL and UL regex are different 
+
+				if ('dlsch_rounds' in dlsch_ulsch_stats) and ('d_retx_th' in checkers):
+					tmp=re.match(r'^.*dlsch_rounds\s+(\d+)\/(\d+)\/(\d+)\/(\d+),\s+dlsch_errors\s+(\d+)',dlsch_ulsch_stats['dlsch_rounds'])
+					if tmp is not None :
+						#captures the different groups from the regex
+						retx_data=[float(x) for x in tmp.groups()]
+						for i in range(0,len(d_perc_retx)):
+							#case where numerator > denumerator with denum ==0 is disregarded, cannot hapen in principle, will lead to 0%
+							d_perc_retx[i] = 0 if (retx_data[i] == 0)  else 100*retx_data[i+1]/retx_data[i]
+							#treating % > 100 , % > requirement
+							if (d_perc_retx[i] > 100) or (d_perc_retx[i] > checkers['d_retx_th'][i]): dlsch_checker_status[i] = -1
+					if -1 in dlsch_checker_status:
+						DLRetxIssue = True
+
+				if ('ulsch_rounds' in dlsch_ulsch_stats)  and ('u_retx_th' in checkers):
+					tmp=re.match(r'^.*ulsch_rounds\s+(\d+)\/(\d+)\/(\d+)\/(\d+),\s+.*,\s+ulsch_errors\s+(\d+)',dlsch_ulsch_stats['ulsch_rounds'])
+					if tmp is not None :
+						retx_data=[float(x) for x in tmp.groups()]
+						for i in range(0,len(d_perc_retx)):
+							u_perc_retx[i] = 0 if (retx_data[i] == 0) else 100*retx_data[i+1]/retx_data[i]
+							if (u_perc_retx[i] > 100) or (u_perc_retx[i] > checkers['u_retx_th'][i]): ulsch_checker_status[i] = -1
+					if -1 in ulsch_checker_status:
+						ULRetxIssue = True
+
+
+
 			#real time statistics
 			datalog_rt_stats['Data']={}
 			if len(real_time_stats)!=0: #check if dictionary is not empty
@@ -1174,6 +1219,22 @@ class RANManagement():
 			logging.debug(statMsg)
 			htmleNBFailureMsg += htmlMsg			
 
+		if DLRetxIssue:
+			retx_checker_status_str = ''
+			for status in dlsch_checker_status : retx_checker_status_str+=str(status)+ ' '
+			logging.debug('\u001B[1;37;41m ' + nodeB_prefix + 'NB ended with too many retransmissions / errors issue in DL ! \u001B[0m')
+			logging.debug('\u001B[1;37;41m Status : ' + retx_checker_status_str + ' \u001B[0m')
+			htmleNBFailureMsg += 'Fail due to retransmissions / errors issue in DL, status : ' + retx_checker_status_str + '\n'
+			global_status = CONST.ENB_RETX_ISSUE
+
+		if ULRetxIssue:
+			retx_checker_status_str = ''
+			for status in ulsch_checker_status : retx_checker_status_str+=str(status)+ ' '
+			logging.debug('\u001B[1;37;41m ' + nodeB_prefix + 'NB ended with too many retransmissions / errors issue in UL ! \u001B[0m')
+			logging.debug('\u001B[1;37;41m Status : ' + retx_checker_status_str + ' \u001B[0m')
+			htmleNBFailureMsg += 'Fail due to retransmissions / errors issue in UL, status : ' + retx_checker_status_str + '\n'
+			global_status = CONST.ENB_RETX_ISSUE
+
 		if RealTimeProcessingIssue:
 			logging.debug('\u001B[1;37;41m ' + nodeB_prefix + 'NB ended with real time processing issue! \u001B[0m')
 			htmleNBFailureMsg += 'Fail due to real time processing issue\n'

ci-scripts/xml_files/container_5g_f1_rfsim.xml

@@ -116,6 +116,8 @@
                 <class>UndeployGenObject</class>
                 <desc>Undeploy all OAI 5G stack</desc>
                 <yaml_path>yaml_files/5g_f1_rfsimulator</yaml_path>
+                <d_retx_th>5,0,0,0</d_retx_th>
+                <u_retx_th>5,0,0,0</u_retx_th>
         </testCase>
 
 </testCaseList>

ci-scripts/xml_files/container_5g_fdd_rfsim.xml

@@ -111,6 +111,8 @@
                 <class>UndeployGenObject</class>
                 <desc>Undeploy all OAI 5G stack</desc>
                 <yaml_path>yaml_files/5g_fdd_rfsimulator</yaml_path>
+                <d_retx_th>5,0,0,0</d_retx_th>
+                <u_retx_th>5,0,0,0</u_retx_th>
         </testCase>
 
 </testCaseList>

ci-scripts/xml_files/container_5g_rfsim.xml

@@ -109,6 +109,8 @@
                 <class>UndeployGenObject</class>
                 <desc>Undeploy all OAI 5G stack</desc>
                 <yaml_path>yaml_files/5g_rfsimulator</yaml_path>
+                <d_retx_th>5,0,0,0</d_retx_th>
+                <u_retx_th>5,0,0,0</u_retx_th>
         </testCase>
 
 </testCaseList>