Merge remote-tracking branch 'origin/ci_rtmon_per_test' into integration_2022_wk04_a

ci-scripts/constants.py

@@ -44,6 +44,7 @@ ENB_PROCESS_ASSERTION = -12
 ENB_PROCESS_REALTIME_ISSUE = -13
 ENB_PROCESS_NOLOGFILE_TO_ANALYZE = -14
 ENB_PROCESS_SLAVE_RRU_NOT_SYNCED = -15
+ENB_REAL_TIME_PROCESSING_ISSUE = -16
 HSS_PROCESS_FAILED = -2
 HSS_PROCESS_OK = +2
 MME_PROCESS_FAILED = -3

ci-scripts/datalog_rt_stats.2x2.yaml

+#this is a configuration file
+#used to build real time processing statistics 
+#for 5G NR phy test (gNB terminate)
+Title : Processing Time (us)
+ColNames :
+  - Metric
+  - Average
+  - Max
+  - Average vs Reference Deviation (Reference Value ; Acceptability Threshold)
+Ref : 
+  feprx : 120.0
+  feptx_prec : 8.0
+  feptx_ofdm : 50.0
+  feptx_total : 75.0
+  L1 Tx processing thread 0 : 300.0
+  L1 Tx processing thread 1 : 300.0
+  DLSCH encoding : 230.0
+  L1 Rx processing : 175.0
+  PUSCH inner-receiver : 100.0
+  PUSCH decoding : 180.0 
+  DL & UL scheduling timing stats : 37.0
+  UL Indication : 38.0
+Threshold :
+  feprx : 1.25
+  feptx_prec : 1.25
+  feptx_ofdm : 1.25
+  feptx_total : 1.25
+  L1 Tx processing thread 0 : 1.25
+  L1 Tx processing thread 1 : 1.25
+  DLSCH encoding : 1.25
+  L1 Rx processing : 1.25
+  PUSCH inner-receiver : 1.25
+  PUSCH decoding : 1.25
+  DL & UL scheduling timing stats : 1.25
+  UL Indication : 1.25

ci-scripts/main.py

@@ -155,6 +155,11 @@ def GetParametersFromXML(action):
 	elif action == 'Initialize_eNB':
 		RAN.eNB_Trace=test.findtext('eNB_Trace')
 		RAN.eNB_Stats=test.findtext('eNB_Stats')
+		datalog_rt_stats_file=test.findtext('rt_stats_cfg')
+		if datalog_rt_stats_file is None:
+			RAN.datalog_rt_stats_file='datalog_rt_stats.default.yaml'
+		else:
+			RAN.datalog_rt_stats_file=datalog_rt_stats_file
 		RAN.Initialize_eNB_args=test.findtext('Initialize_eNB_args')
 		eNB_instance=test.findtext('eNB_instance')
 		USRPIPAddress=test.findtext('USRP_IPAddress')

ci-scripts/ran.py

@@ -92,6 +92,7 @@ class RANManagement():
 		self.epcPcapFile = ''
 		self.runtime_stats= ''
 		self.datalog_rt_stats={}
+		self.datalog_rt_stats_file='datalog_rt_stats.default.yaml'
 		self.eNB_Trace = '' #if 'yes', Tshark will be launched at initialization
 		self.eNB_Stats = '' #if 'yes', Statistics Monitor will be launched at initialization		
 		self.USRPIPAddress = ''
@@ -722,6 +723,9 @@ class RANManagement():
 				logStatus = self.AnalyzeLogFile_eNB(fileToAnalyze, HTML)
 				if (logStatus < 0):
 					HTML.CreateHtmlTestRow('N/A', 'KO', logStatus)
+					#display rt stats for gNB only
+					if len(self.datalog_rt_stats)!=0 and nodeB_prefix == 'g':
+						HTML.CreateHtmlDataLogTable(self.datalog_rt_stats)
 					self.prematureExit = True
 					self.eNBmbmsEnables[int(self.eNB_instance)] = False
 					return
@@ -810,6 +814,7 @@ class RANManagement():
 		#NSA specific log markers
 		nsa_markers ={'SgNBReleaseRequestAcknowledge': [],'FAILURE': [], 'scgFailureInformationNR-r15': [], 'SgNBReleaseRequest': []}
 		nodeB_prefix_found = False
+		RealTimeProcessingIssue = False
 	
 		line_cnt=0 #log file line counter
 		for line in enb_log_file.readlines():
@@ -1011,7 +1016,7 @@ class RANManagement():
 		#the following part takes the *_stats.log files as source (not the stdout log file)
 
 		#the datalog config file has to be loaded
-		datalog_rt_stats_file='datalog_rt_stats.yaml'
+		datalog_rt_stats_file=self.datalog_rt_stats_file
 		if (os.path.isfile(datalog_rt_stats_file)):
 			yaml_file=datalog_rt_stats_file
 		elif (os.path.isfile('ci-scripts/'+datalog_rt_stats_file)):
@@ -1127,8 +1132,7 @@ class RANManagement():
 				#check if there is a fail => will render the test as failed
 				for k in datalog_rt_stats['Data']:
 					if float(datalog_rt_stats['Data'][k][2])> datalog_rt_stats['Threshold'][k]: #condition for fail : avg/ref is greater than the fixed threshold
-						#setting prematureExit is ok although not the best option
-						self.prematureExit=False #temp for debug : do not stop the test if RT stats are excedeed
+						RealTimeProcessingIssue = True
 			else:
 				statMsg = 'No real time stats found in the log file\n'
 				logging.debug('No real time stats found in the log file')
@@ -1162,7 +1166,10 @@ class RANManagement():
 			logging.debug(statMsg)
 			htmleNBFailureMsg += htmlMsg			
 
-
+		if RealTimeProcessingIssue:
+			logging.debug('\u001B[1;37;41m ' + nodeB_prefix + 'NB ended with real time processing issue! \u001B[0m')
+			htmleNBFailureMsg += 'Fail due to real time processing issue\n'
+			global_status = CONST.ENB_REAL_TIME_PROCESSING_ISSUE
 		if uciStatMsgCount > 0:
 			statMsg = nodeB_prefix + 'NB showed ' + str(uciStatMsgCount) + ' "uci->stat" message(s)'
 			logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')

ci-scripts/xml_files/fr1_nsa_2x2_quectel.xml

@@ -79,6 +79,7 @@
 		<eNB_serverId>1</eNB_serverId>
 		<air_interface>nr</air_interface>
 		<eNB_Stats>yes</eNB_Stats>
+		<rt_stats_cfg>datalog_rt_stats.2x2.yaml</rt_stats_cfg>
 		<USRP_IPAddress>192.168.18.240</USRP_IPAddress>
 	</testCase>
 

ci-scripts/xml_files/fr1_nsa_2x2_quectel_attach_detach.xml

@@ -99,6 +99,7 @@
 		<eNB_serverId>1</eNB_serverId>
 		<air_interface>nr</air_interface>
 		<eNB_Stats>yes</eNB_Stats>
+		<rt_stats_cfg>datalog_rt_stats.2x2.yaml</rt_stats_cfg>
 		<USRP_IPAddress>192.168.18.240</USRP_IPAddress>
 	</testCase>
 

ci-scripts/xml_files/fr1_sa_quectel.xml

@@ -64,6 +64,7 @@
 		<air_interface>nr</air_interface>
 		<eNB_Trace>yes</eNB_Trace>
 		<eNB_Stats>yes</eNB_Stats>
+		<rt_stats_cfg>datalog_rt_stats.2x2.yaml</rt_stats_cfg>
 		<USRP_IPAddress>192.168.18.240</USRP_IPAddress>
 	</testCase>
 

ci-scripts/xml_files/fr1_sa_quectel_162prb.xml

@@ -64,6 +64,7 @@
 		<air_interface>nr</air_interface>
 		<eNB_Trace>yes</eNB_Trace>
 		<eNB_Stats>yes</eNB_Stats>
+		<rt_stats_cfg>datalog_rt_stats.2x2.yaml</rt_stats_cfg>
 		<USRP_IPAddress>192.168.18.240</USRP_IPAddress>
 	</testCase>
 

ci-scripts/xml_files/fr1_sa_quectel_stages.xml

@@ -71,6 +71,7 @@
 		<air_interface>nr</air_interface>
 		<eNB_Trace>yes</eNB_Trace>
 		<eNB_Stats>yes</eNB_Stats>
+		<rt_stats_cfg>datalog_rt_stats.2x2.yaml</rt_stats_cfg>
 		<USRP_IPAddress>192.168.18.240</USRP_IPAddress>
 	</testCase>
 

ci-scripts/xml_files/fr1_sa_quectel_stages_162prb.xml

@@ -73,6 +73,7 @@
 		<air_interface>nr</air_interface>
 		<eNB_Trace>yes</eNB_Trace>
 		<eNB_Stats>yes</eNB_Stats>
+		<rt_stats_cfg>datalog_rt_stats.2x2.yaml</rt_stats_cfg>
 		<USRP_IPAddress>192.168.18.240</USRP_IPAddress>
 	</testCase>