Merge remote-tracking branch 'origin/add_oai_cn_docker_support_to_ci' into...

Merge remote-tracking branch 'origin/add_oai_cn_docker_support_to_ci' into develop_integration_2020_w22

ci-scripts/main.py

@@ -403,23 +403,50 @@ class OaiCiTest():
 				if (outterLoopCounter == 0):
 					doOutterLoop = False
 				continue
+			# We are now checking if sync w/ eNB DOES NOT OCCUR
+			# Usually during the cell synchronization stage, the UE returns with No cell synchronization message
+			# That is the case for LTE
+			# In NR case, it's a positive message that will show if synchronization occurs
+			doLoop = True
 			if RAN.Getair_interface() == 'nr':
-				fullSyncStatus = True
-				doOutterLoop = False
+				loopCounter = 10
 			else:
 				# We are now checking if sync w/ eNB DOES NOT OCCUR
 				# Usually during the cell synchronization stage, the UE returns with No cell synchronization message
-				doLoop = True
 				loopCounter = 10
-				while (doLoop):
-					loopCounter = loopCounter - 1
-					if (loopCounter == 0):
+			while (doLoop):
+				loopCounter = loopCounter - 1
+				if (loopCounter == 0):
+					if RAN.Getair_interface() == 'nr':
+						# Here we do have great chances that UE did NOT cell-sync w/ gNB
+						doLoop = False
+						fullSyncStatus = False
+						logging.debug('Never seen the NR-Sync message (Measured Carrier Frequency) --> try again')
+						time.sleep(6)
+						# Stopping the NR-UE  
+						SSH.command('ps -aux | grep --text --color=never softmodem | grep -v grep', '\$', 4)
+						result = re.search('nr-uesoftmodem', SSH.getBefore())
+						if result is not None:
+							SSH.command('echo ' + self.UEPassword + ' | sudo -S killall --signal=SIGINT nr-uesoftmodem', '\$', 4)
+						time.sleep(6)
+					else:
 						# Here we do have a great chance that the UE did cell-sync w/ eNB
 						doLoop = False
 						doOutterLoop = False
 						fullSyncStatus = True
 						continue
-					SSH.command('stdbuf -o0 cat ue_' + self.testCase_id + '.log | egrep --text --color=never -i "wait|sync"', '\$', 4)
+				SSH.command('stdbuf -o0 cat ue_' + self.testCase_id + '.log | egrep --text --color=never -i "wait|sync|Frequency"', '\$', 4)
+				if RAN.Getair_interface() == 'nr':
+					# Positive messaging -->
+					result = re.search('Measured Carrier Frequency', SSH.getBefore())
+					if result is not None:
+						doLoop = False
+						doOutterLoop = False
+						fullSyncStatus = True
+					else:
+						time.sleep(6)
+				else:
+					# Negative messaging -->
 					result = re.search('No cell synchronization found', SSH.getBefore())
 					if result is None:
 						time.sleep(6)
@@ -432,13 +459,22 @@ class OaiCiTest():
 						result = re.search('lte-uesoftmodem', SSH.getBefore())
 						if result is not None:
 							SSH.command('echo ' + self.UEPassword + ' | sudo -S killall --signal=SIGINT lte-uesoftmodem', '\$', 4)
-				outterLoopCounter = outterLoopCounter - 1
-				if (outterLoopCounter == 0):
-					doOutterLoop = False
+			outterLoopCounter = outterLoopCounter - 1
+			if (outterLoopCounter == 0):
+				doOutterLoop = False
 
-		if fullSyncStatus and gotSyncStatus and RAN.Getair_interface() == 'lte':
-			result = re.search('--no-L2-connect', str(self.Initialize_OAI_UE_args))
-			if result is None:
+		if fullSyncStatus and gotSyncStatus:
+			doInterfaceCheck = False
+			if RAN.Getair_interface() == 'lte':
+				result = re.search('--no-L2-connect', str(self.Initialize_OAI_UE_args))
+				if result is None:
+					doInterfaceCheck = True
+			# For the moment, only in explicit noS1 without kernel module (ie w/ tunnel interface)
+			if RAN.Getair_interface() == 'nr':
+				result = re.search('--noS1 --nokrnmod 1', str(self.Initialize_OAI_UE_args))
+				if result is not None:
+					doInterfaceCheck = True
+			if doInterfaceCheck:
 				SSH.command('ifconfig oaitun_ue1', '\$', 4)
 				SSH.command('ifconfig oaitun_ue1', '\$', 4)
 				# ifconfig output is different between ubuntu 16 and ubuntu 18
@@ -3316,10 +3352,13 @@ while len(argvs) > 1:
 		EPC.SetSourceCodePath(matchReg.group(1))
 	elif re.match('^\-\-EPCType=(.+)$', myArgv, re.IGNORECASE):
 		matchReg = re.match('^\-\-EPCType=(.+)$', myArgv, re.IGNORECASE)
-		if re.match('OAI', matchReg.group(1), re.IGNORECASE) or re.match('ltebox', matchReg.group(1), re.IGNORECASE) or re.match('OAI-Rel14-CUPS', matchReg.group(1), re.IGNORECASE):
+		if re.match('OAI', matchReg.group(1), re.IGNORECASE) or re.match('ltebox', matchReg.group(1), re.IGNORECASE) or re.match('OAI-Rel14-CUPS', matchReg.group(1), re.IGNORECASE) or re.match('OAI-Rel14-Docker', matchReg.group(1), re.IGNORECASE):
 			EPC.SetType(matchReg.group(1))
 		else:
-			sys.exit('Invalid EPC Type: ' + matchReg.group(1) + ' -- (should be OAI or ltebox or OAI-Rel14-CUPS)')
+			sys.exit('Invalid EPC Type: ' + matchReg.group(1) + ' -- (should be OAI or ltebox or OAI-Rel14-CUPS or OAI-Rel14-Docker)')
+	elif re.match('^\-\-EPCContainerPrefix=(.+)$', myArgv, re.IGNORECASE):
+		matchReg = re.match('^\-\-EPCContainerPrefix=(.+)$', myArgv, re.IGNORECASE)
+		EPC.SetContainerPrefix(matchReg.group(1))
 	elif re.match('^\-\-ADBIPAddress=(.+)$', myArgv, re.IGNORECASE):
 		matchReg = re.match('^\-\-ADBIPAddress=(.+)$', myArgv, re.IGNORECASE)
 		CiTestObj.ADBIPAddress = matchReg.group(1)
@@ -3542,6 +3581,7 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re
 			sys.exit(1)
 	if (EPC.GetIPAddress() != '') and (EPC.GetIPAddress() != 'none'):
 		CiTestObj.CheckFlexranCtrlInstallation()
+		EPC.SetMmeIPAddress()
 
 	#get the list of tests to be done
 	todo_tests=[]

ci-scripts/ran.py

@@ -522,8 +522,8 @@ class RANManagement():
 		# Make a copy and adapt to EPC / eNB IP addresses
 		mySSH.command('cp ' + full_config_file + ' ' + ci_full_config_file, '\$', 5)
 		if self.epcObj is not None:
-			localEpcIpAddr = self.epcObj.GetIPAddress()
-			mySSH.command('sed -i -e \'s/CI_MME_IP_ADDR/' + localEpcIpAddr + '/\' ' + ci_full_config_file, '\$', 2);
+			localMmeIpAddr = self.epcObj.GetMmeIPAddress()
+			mySSH.command('sed -i -e \'s/CI_MME_IP_ADDR/' + localMmeIpAddr + '/\' ' + ci_full_config_file, '\$', 2);
 		mySSH.command('sed -i -e \'s/CI_ENB_IP_ADDR/' + lIpAddr + '/\' ' + ci_full_config_file, '\$', 2);
 		mySSH.command('sed -i -e \'s/CI_RCC_IP_ADDR/' + self.eNBIPAddress + '/\' ' + ci_full_config_file, '\$', 2);
 		mySSH.command('sed -i -e \'s/CI_RRU1_IP_ADDR/' + self.eNB1IPAddress + '/\' ' + ci_full_config_file, '\$', 2);
@@ -801,6 +801,7 @@ class RANManagement():
 		pdcpFailure = 0
 		ulschFailure = 0
 		ulschReceiveOK = 0
+		gnbRxTxWakeUpFailure = 0
 		cdrxActivationMessageCount = 0
 		dropNotEnoughRBs = 0
 		mbmsRequestMsg = 0
@@ -910,7 +911,10 @@ class RANManagement():
 			result = re.search('PDCP.*Out of Resources.*reason', str(line))
 			if result is not None:
 				pdcpFailure += 1
-			result = re.search('ULSCH in error in round', str(line))
+			result = re.search('could not wakeup gNB rxtx process', str(line))
+			if result is not None:
+				gnbRxTxWakeUpFailure += 1
+			result = re.search('ULSCH in error in round|ULSCH 0 in error', str(line))
 			if result is not None:
 				ulschFailure += 1
 			result = re.search('ULSCH received ok', str(line))
@@ -940,6 +944,10 @@ class RANManagement():
 				statMsg = nodeB_prefix + 'NB showed ' + str(ulschReceiveOK) + ' "ULSCH received ok" message(s)'
 				logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
 				htmleNBFailureMsg += statMsg + '\n'
+			if gnbRxTxWakeUpFailure > 0:
+				statMsg = nodeB_prefix + 'NB showed ' + str(gnbRxTxWakeUpFailure) + ' "could not wakeup gNB rxtx process" message(s)'
+				logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
+				htmleNBFailureMsg += statMsg + '\n'
 		if uciStatMsgCount > 0:
 			statMsg = nodeB_prefix + 'NB showed ' + str(uciStatMsgCount) + ' "uci->stat" message(s)'
 			logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')

ci-scripts/sshconnection.py

@@ -52,15 +52,19 @@ class SSHConnection():
 		self.picocom_closure = True
 
 	def open(self, ipaddress, username, password):
+		extraSshOptions = ''
 		count = 0
 		connect_status = False
+		if ipaddress == '192.168.18.197':
+			extraSshOptions = ' -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'
 		while count < 4:
-			self.ssh = pexpect.spawn('ssh', [username + '@' + ipaddress], timeout = 5)
+			self.ssh = pexpect.spawn('ssh', [username + '@' + ipaddress + extraSshOptions], timeout = 5)
 			self.sshresponse = self.ssh.expect(['Are you sure you want to continue connecting (yes/no)?', 'password:', 'Last login', pexpect.EOF, pexpect.TIMEOUT])
 			if self.sshresponse == 0:
 				self.ssh.sendline('yes')
-				self.ssh.expect('password:')
-				self.ssh.sendline(password)
+				self.sshresponse = self.ssh.expect(['password:', username + '@'])
+				if self.sshresponse == 0:
+					self.ssh.sendline(password)
 				self.sshresponse = self.ssh.expect(['\$', 'Permission denied', 'password:', pexpect.EOF, pexpect.TIMEOUT])
 				if self.sshresponse == 0:
 					count = 10