Merge remote-tracking branch 'origin/ci_improve_pass_fail' into integration_2021_wk51_a

ci-scripts/cls_oaicitest.py

@@ -126,6 +126,7 @@ class OaiCiTest():
 		self.desc = ''
 		self.ping_args = ''
 		self.ping_packetloss_threshold = ''
+		self.ping_rttavg_threshold =''
 		self.iperf_args = ''
 		self.iperf_packetloss_threshold = ''
 		self.iperf_profile = ''
@@ -1632,6 +1633,7 @@ class OaiCiTest():
 			min_msg = 'RTT(Min)    : ' + rtt_min + ' ms'
 			avg_msg = 'RTT(Avg)    : ' + rtt_avg + ' ms'
 			max_msg = 'RTT(Max)    : ' + rtt_max + ' ms'
+
 			lock.acquire()
 			logging.debug('\u001B[1;37;44m ping result (' + UE_IPAddress + ') \u001B[0m')
 			logging.debug('\u001B[1;34m    ' + pal_msg + '\u001B[0m')
@@ -1658,17 +1660,29 @@ class OaiCiTest():
 				ping_stat_msg+='RTT(Max)    : ' + str("{:.2f}".format(ping_stat['max_1'])) + 'ms \n'
 
 			#building html message
-			qMsg = pal_msg + '\n' + min_msg + '\n' + avg_msg + '\n' + max_msg + '\n' + ping_stat_msg
+			qMsg = pal_msg + '\n' + min_msg + '\n' + avg_msg + '\n' + max_msg + '\n'  + ping_stat_msg
+
+			#checking packet loss compliance
 			packetLossOK = True
 			if packetloss is not None:
 				if float(packetloss) > float(self.ping_packetloss_threshold):
 					qMsg += '\nPacket Loss too high'
-					logging.debug('\u001B[1;37;41m Packet Loss too high \u001B[0m')
+					logging.debug('\u001B[1;37;41m Packet Loss too high; Target: '+ self.ping_packetloss_threshold + '%\u001B[0m')
 					packetLossOK = False
 				elif float(packetloss) > 0:
 					qMsg += '\nPacket Loss is not 0%'
 					logging.debug('\u001B[1;30;43m Packet Loss is not 0% \u001B[0m')
-			if (packetLossOK):
+
+			#checking RTT avg compliance
+			rttavgOK = True
+			if self.ping_rttavg_threshold != '':
+				if float(rtt_avg)>float(self.ping_rttavg_threshold):
+					ping_rttavg_error_msg = 'RTT(Avg) too high: ' + rtt_avg + ' ms; Target: '+ self.ping_rttavg_threshold+ ' ms'
+					qMsg += '\n'+ping_rttavg_error_msg
+					logging.debug('\u001B[1;37;41m'+ ping_rttavg_error_msg +' \u001B[0m')
+					rttavgOK = False
+
+			if packetLossOK and rttavgOK:
 				statusQueue.put(0)
 			else:
 				statusQueue.put(-1)

ci-scripts/main.py

@@ -295,6 +295,11 @@ def GetParametersFromXML(action):
 			CiTestObj.ue_id = ""
 		else:
 			CiTestObj.ue_id = ue_id
+		ping_rttavg_threshold = test.findtext('ping_rttavg_threshold')
+		if (ping_rttavg_threshold is None):
+			CiTestObj.ping_rttavg_threshold = ""
+		else:
+			CiTestObj.ping_rttavg_threshold = ping_rttavg_threshold
 
 	elif action == 'Iperf':
 		CiTestObj.iperf_args = test.findtext('iperf_args')

ci-scripts/ran.py

@@ -752,6 +752,7 @@ class RANManagement():
 		msgLine = 0
 		foundSegFault = False
 		foundRealTimeIssue = False
+		foundRealTimeIssue_cnt = 0
 		rrcSetupComplete = 0
 		rrcReleaseRequest = 0
 		rrcReconfigRequest = 0
@@ -890,6 +891,7 @@ class RANManagement():
 			result = re.search('LLL', str(line))
 			if result is not None and not exitSignalReceived:
 				foundRealTimeIssue = True
+				foundRealTimeIssue_cnt += 1
 			if foundAssertion and (msgLine < 3):
 				msgLine += 1
 				msgAssertion += str(line)
@@ -1258,7 +1260,7 @@ class RANManagement():
 			global_status = CONST.ENB_PROCESS_ASSERTION
 		if foundRealTimeIssue:
 			logging.debug('\u001B[1;37;41m ' + nodeB_prefix + 'NB faced real time issues! \u001B[0m')
-			htmleNBFailureMsg += nodeB_prefix + 'NB faced real time issues!\n'
+			htmleNBFailureMsg += nodeB_prefix + 'NB faced real time issues! COUNT = '+ str(foundRealTimeIssue_cnt) +' lines\n'
 		if rlcDiscardBuffer > 0:
 			rlcMsg = nodeB_prefix + 'NB RLC discarded ' + str(rlcDiscardBuffer) + ' buffer(s)'
 			logging.debug('\u001B[1;37;41m ' + rlcMsg + ' \u001B[0m')

ci-scripts/xml_files/container_nsa_b200_quectel.xml

@@ -39,6 +39,8 @@
  050001
  070000
  070001
+ 050000
+ 050001
  010002
  000001
  030202
@@ -103,6 +105,7 @@
 		<id>idefix</id>
 		<ping_args>-c 20</ping_args>
 		<ping_packetloss_threshold>1</ping_packetloss_threshold>
+		<ping_rttavg_threshold>15</ping_rttavg_threshold>
 	</testCase>
 
 	<testCase id="050001">
@@ -111,6 +114,7 @@
 		<id>idefix</id>
 		<ping_args>-c 100 -i 0.2</ping_args>
 		<ping_packetloss_threshold>1</ping_packetloss_threshold>
+		<ping_rttavg_threshold>15</ping_rttavg_threshold>
 	</testCase>
 
 	<testCase id="070000">

ci-scripts/xml_files/fr1_lte_2x2_quectel.xml

@@ -87,14 +87,16 @@
 		<desc>Ping: 20 pings</desc>
 		<id>nrmodule2_quectel</id>
 		<ping_args>-c 20</ping_args>
-		<ping_packetloss_threshold>5</ping_packetloss_threshold>
+		<ping_packetloss_threshold>1</ping_packetloss_threshold>
+		<ping_rttavg_threshold>40</ping_rttavg_threshold>
 	</testCase>
 	<testCase id="050001">
 		<class>Ping</class>
 		<desc>Ping: 100 pings, size 1024</desc>
 		<id>nrmodule2_quectel</id>
 		<ping_args>-c 100 -s 1024 -i 0,2</ping_args>
-		<ping_packetloss_threshold>5</ping_packetloss_threshold>
+		<ping_packetloss_threshold>1</ping_packetloss_threshold>
+		<ping_rttavg_threshold>30</ping_rttavg_threshold>
 	</testCase>
 
 

ci-scripts/xml_files/fr1_nsa_quectel.xml

@@ -38,6 +38,8 @@
  000001
  070001
  000001
+ 050000
+ 050001
  010002
  080001
  080000
@@ -98,6 +100,7 @@
 		<id>idefix</id>
 		<ping_args>-c 20</ping_args>
 		<ping_packetloss_threshold>1</ping_packetloss_threshold>
+		<ping_rttavg_threshold>15</ping_rttavg_threshold>
 	</testCase>
 
 	<testCase id="050001">
@@ -106,6 +109,7 @@
 		<id>idefix</id>
 		<ping_args>-c 100 -i 0.2</ping_args>
 		<ping_packetloss_threshold>1</ping_packetloss_threshold>
+		<ping_rttavg_threshold>15</ping_rttavg_threshold>
 	</testCase>
 
 	<testCase id="070000">

ci-scripts/xml_files/fr1_sa_quectel.xml

@@ -85,7 +85,8 @@
 		<desc>Ping: 20pings in 20sec</desc>
 		<id>nrmodule2_quectel</id>
 		<ping_args>-c 20</ping_args>
-		<ping_packetloss_threshold>5</ping_packetloss_threshold>
+		<ping_packetloss_threshold>1</ping_packetloss_threshold>
+		<ping_rttavg_threshold>15</ping_rttavg_threshold>
 	</testCase>
 
 	<testCase id="050001">
@@ -93,7 +94,8 @@
 		<desc>Ping: 100pings in 20sec</desc>
 		<id>nrmodule2_quectel</id>
 		<ping_args>-c 100 -i 0,2</ping_args>
-		<ping_packetloss_threshold>5</ping_packetloss_threshold>
+		<ping_packetloss_threshold>1</ping_packetloss_threshold>
+		<ping_rttavg_threshold>15</ping_rttavg_threshold>
 	</testCase>
 
 	<testCase id="070000">

ci-scripts/xml_files/fr1_sa_quectel_stages.xml

@@ -92,7 +92,8 @@
 		<desc>Ping: 20pings in 20sec</desc>
 		<id>nrmodule2_quectel</id>
 		<ping_args>-c 20</ping_args>
-		<ping_packetloss_threshold>5</ping_packetloss_threshold>
+		<ping_packetloss_threshold>1</ping_packetloss_threshold>
+		<ping_rttavg_threshold>15</ping_rttavg_threshold>
 	</testCase>