CI: check the NR-UE cell-synchronization during launch not and just in post-mortem diagnosis

Signed-off-by: Raphael Defosseux <raphael.defosseux@eurecom.fr>

ci-scripts/main.py

@@ -403,23 +403,50 @@ class OaiCiTest():
 				if (outterLoopCounter == 0):
 					doOutterLoop = False
 				continue
+			# We are now checking if sync w/ eNB DOES NOT OCCUR
+			# Usually during the cell synchronization stage, the UE returns with No cell synchronization message
+			# That is the case for LTE
+			# In NR case, it's a positive message that will show if synchronization occurs
+			doLoop = True
 			if RAN.Getair_interface() == 'nr':
-				fullSyncStatus = True
-				doOutterLoop = False
+				loopCounter = 10
 			else:
 				# We are now checking if sync w/ eNB DOES NOT OCCUR
 				# Usually during the cell synchronization stage, the UE returns with No cell synchronization message
-				doLoop = True
 				loopCounter = 10
-				while (doLoop):
-					loopCounter = loopCounter - 1
-					if (loopCounter == 0):
+			while (doLoop):
+				loopCounter = loopCounter - 1
+				if (loopCounter == 0):
+					if RAN.Getair_interface() == 'nr':
+						# Here we do have great chances that UE did NOT cell-sync w/ gNB
+						doLoop = False
+						fullSyncStatus = False
+						logging.debug('Never seen the NR-Sync message (Measured Carrier Frequency) --> try again')
+						time.sleep(6)
+						# Stopping the NR-UE  
+						SSH.command('ps -aux | grep --text --color=never softmodem | grep -v grep', '\$', 4)
+						result = re.search('nr-uesoftmodem', SSH.getBefore())
+						if result is not None:
+							SSH.command('echo ' + self.UEPassword + ' | sudo -S killall --signal=SIGINT nr-uesoftmodem', '\$', 4)
+						time.sleep(6)
+					else:
 						# Here we do have a great chance that the UE did cell-sync w/ eNB
 						doLoop = False
 						doOutterLoop = False
 						fullSyncStatus = True
 						continue
-					SSH.command('stdbuf -o0 cat ue_' + self.testCase_id + '.log | egrep --text --color=never -i "wait|sync"', '\$', 4)
+				SSH.command('stdbuf -o0 cat ue_' + self.testCase_id + '.log | egrep --text --color=never -i "wait|sync|Frequency"', '\$', 4)
+				if RAN.Getair_interface() == 'nr':
+					# Positive messaging -->
+					result = re.search('Measured Carrier Frequency', SSH.getBefore())
+					if result is not None:
+						doLoop = False
+						doOutterLoop = False
+						fullSyncStatus = True
+					else:
+						time.sleep(6)
+				else:
+					# Negative messaging -->
 					result = re.search('No cell synchronization found', SSH.getBefore())
 					if result is None:
 						time.sleep(6)
@@ -432,13 +459,22 @@ class OaiCiTest():
 						result = re.search('lte-uesoftmodem', SSH.getBefore())
 						if result is not None:
 							SSH.command('echo ' + self.UEPassword + ' | sudo -S killall --signal=SIGINT lte-uesoftmodem', '\$', 4)
-				outterLoopCounter = outterLoopCounter - 1
-				if (outterLoopCounter == 0):
-					doOutterLoop = False
+			outterLoopCounter = outterLoopCounter - 1
+			if (outterLoopCounter == 0):
+				doOutterLoop = False
 
-		if fullSyncStatus and gotSyncStatus and RAN.Getair_interface() == 'lte':
-			result = re.search('--no-L2-connect', str(self.Initialize_OAI_UE_args))
-			if result is None:
+		if fullSyncStatus and gotSyncStatus:
+			doInterfaceCheck = False
+			if RAN.Getair_interface() == 'lte':
+				result = re.search('--no-L2-connect', str(self.Initialize_OAI_UE_args))
+				if result is None:
+					doInterfaceCheck = True
+			# For the moment, only in explicit noS1 without kernel module (ie w/ tunnel interface)
+			if RAN.Getair_interface() == 'nr':
+				result = re.search('--noS1 --nokrnmod 1', str(self.Initialize_OAI_UE_args))
+				if result is not None:
+					doInterfaceCheck = True
+			if doInterfaceCheck:
 				SSH.command('ifconfig oaitun_ue1', '\$', 4)
 				SSH.command('ifconfig oaitun_ue1', '\$', 4)
 				# ifconfig output is different between ubuntu 16 and ubuntu 18

ci-scripts/ran.py

@@ -801,6 +801,7 @@ class RANManagement():
 		pdcpFailure = 0
 		ulschFailure = 0
 		ulschReceiveOK = 0
+		gnbRxTxWakeUpFailure = 0
 		cdrxActivationMessageCount = 0
 		dropNotEnoughRBs = 0
 		mbmsRequestMsg = 0
@@ -910,7 +911,10 @@ class RANManagement():
 			result = re.search('PDCP.*Out of Resources.*reason', str(line))
 			if result is not None:
 				pdcpFailure += 1
-			result = re.search('ULSCH in error in round', str(line))
+			result = re.search('could not wakeup gNB rxtx process', str(line))
+			if result is not None:
+				gnbRxTxWakeUpFailure += 1
+			result = re.search('ULSCH in error in round|ULSCH 0 in error', str(line))
 			if result is not None:
 				ulschFailure += 1
 			result = re.search('ULSCH received ok', str(line))
@@ -940,6 +944,10 @@ class RANManagement():
 				statMsg = nodeB_prefix + 'NB showed ' + str(ulschReceiveOK) + ' "ULSCH received ok" message(s)'
 				logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
 				htmleNBFailureMsg += statMsg + '\n'
+			if gnbRxTxWakeUpFailure > 0:
+				statMsg = nodeB_prefix + 'NB showed ' + str(gnbRxTxWakeUpFailure) + ' "could not wakeup gNB rxtx process" message(s)'
+				logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
+				htmleNBFailureMsg += statMsg + '\n'
 		if uciStatMsgCount > 0:
 			statMsg = nodeB_prefix + 'NB showed ' + str(uciStatMsgCount) + ' "uci->stat" message(s)'
 			logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')