Refactor Ping to always handle Module_UE

ci-scripts/cls_oaicitest.py

@@ -51,6 +51,7 @@ import constants as CONST
 import sshconnection
 
 import cls_module_ue
+import cls_cmd
 
 logging.getLogger("matplotlib").setLevel(logging.WARNING)
 import matplotlib.pyplot as plt
@@ -744,30 +745,18 @@ class OaiCiTest():
 		statusQueue.put(message)
 		lock.release()
 
-	def Ping_common(self, lock, UE_IPAddress, device_id, statusQueue,EPC, Module_UE,RAN):
+	def Ping_common(self, lock, statusQueue,EPC, ue, RAN):
 		try:
-			SSH = sshconnection.SSHConnection()
 			# Launch ping on the EPC side (true for ltebox and old open-air-cn)
 			# But for OAI-Rel14-CUPS, we launch from python executor
 			ping_status = 0
-			launchFromEpc = True
-			launchFromModule = False
-			launchFromASUE = False
+			launchFromEpc = False
+			ping_log_file = f'ping_{self.testCase_id}_{ue.getName()}.log'
 			if re.match('OAI-Rel14-CUPS', EPC.Type, re.IGNORECASE):
 				launchFromEpc = False
-			#if module, ping from module to EPC
-			if self.ue_id!='':
-				if (re.match('amarisoft', self.ue_id, re.IGNORECASE)):
-					launchFromEpc = False
-					launchFromASUE = True
-				else:
-					launchFromEpc = False
-					launchFromModule = True
-			#no ping args for ASUE
-			if self.ping_args!='':
-				ping_time = re.findall("-c (\d+)",str(self.ping_args))
-
+			ping_time = re.findall("-c *(\d+)",str(self.ping_args))
 			if launchFromEpc:
+				SSH = sshconnection.SSHConnection()
 				SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password)
 				SSH.command('cd ' + EPC.SourceCodePath, '\$', 5)
 				SSH.command('cd scripts', '\$', 5)
@@ -776,93 +765,59 @@ class OaiCiTest():
 				if (re.match('OAI-Rel14-Docker', EPC.Type, re.IGNORECASE)) or (re.match('OAICN5G', EPC.Type, re.IGNORECASE)):
 					launchFromTrfContainer = True
 				if launchFromTrfContainer:
-					ping_status = SSH.command('docker exec -it prod-trf-gen /bin/bash -c "ping ' + self.ping_args + ' ' + UE_IPAddress + '" 2>&1 | tee ping_' + self.testCase_id + '_' + device_id + '.log', '\$', int(ping_time[0])*1.5)				
+					ping_status = SSH.command(f'docker exec -it prod-trf-gen /bin/bash -c "ping {self.ping_args} {ue.getIP()}" 2>&1 | tee {ping_log_file}', '\$', int(ping_time[0])*1.5)
 				else:
-					ping_status = SSH.command('stdbuf -o0 ping ' + self.ping_args + ' ' + UE_IPAddress + ' 2>&1 | stdbuf -o0 tee ping_' + self.testCase_id + '_' + device_id + '.log', '\$', int(ping_time[0])*1.5)
-				ping_log_file='ping_' + self.testCase_id + '_' + device_id + '.log'
+					ping_status = SSH.command(f'stdbuf -o0 ping {self.ping_args} {ue.getIP()} 2>&1 | stdbuf -o0 tee {ping_log_file}', '\$', int(ping_time[0])*1.5)
 				#copy the ping log file to have it locally for analysis (ping stats)
-				SSH.copyin(EPC.IPAddress, EPC.UserName, EPC.Password, EPC.SourceCodePath + '/scripts/ping_' + self.testCase_id + '_' + device_id + '.log', '.')				
+				SSH.copyin(EPC.IPAddress, EPC.UserName, EPC.Password, EPC.SourceCodePath + '/scripts/{ping_log_file}', '.')				
+				SSH.close()
 			else:
-				if (launchFromModule == False) and (launchFromASUE == False):
-					#ping log file is on the python executor
-					cmd = 'ping ' + self.ping_args + ' ' + UE_IPAddress + ' > ping_' + self.testCase_id + '_' + device_id + '.log 2>&1'
-					message = cmd + '\n'
-					logging.debug(cmd)
-					ret = subprocess.run(cmd, shell=True)
-					ping_status = ret.returncode
-					#copy the ping log file to an other folder for log collection (source and desti				elif (launchfromModule == True) and (launchfromASUE == False): #launch from Modulenation are EPC)
-					SSH.copyout(EPC.IPAddress, EPC.UserName, EPC.Password, 'ping_' + self.testCase_id + '_' + device_id + '.log', EPC.SourceCodePath + '/scripts')
-                	#copy the ping log file to have it locally for analysis (ping stats)
-					logging.debug(EPC.SourceCodePath + 'ping_' + self.testCase_id + '_' + device_id + '.log')
-					SSH.copyin(EPC.IPAddress, EPC.UserName, EPC.Password, EPC.SourceCodePath  +'/scripts/ping_' + self.testCase_id + '_' + device_id + '.log', '.')
-
-					SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password)
-					#cat is executed on EPC
-					SSH.command('cat ' + EPC.SourceCodePath + '/scripts/ping_' + self.testCase_id + '_' + device_id + '.log', '\$', 5)
-					ping_log_file='/scripts/ping_' + self.testCase_id + '_' + device_id + '.log'
-
-				elif (launchFromModule == True) and (launchFromASUE == False): #launch from Module
-					SSH.open(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword)
-					#target address is different depending on EPC type
-					if re.match('OAI-Rel14-Docker', EPC.Type, re.IGNORECASE):
-						Target = EPC.MmeIPAddress
-					elif re.match('OAICN5G', EPC.Type, re.IGNORECASE):
-						Target = EPC.MmeIPAddress
-					else:
-						Target = EPC.IPAddress
-					#ping from module NIC rather than IP address to make sure round trip is over the air	
-					cmd = 'ping -I ' + Module_UE.UENetwork  + ' ' + self.ping_args + ' ' +  Target  + ' > ping_' + self.testCase_id + '_' + self.ue_id + '.log 2>&1'
-					SSH.command(cmd,'\$',int(ping_time[0])*1.5)
-
+				#target address is different depending on EPC type
+				if re.match('OAI-Rel14-Docker', EPC.Type, re.IGNORECASE):
+					Target = EPC.MmeIPAddress
+				elif re.match('OAICN5G', EPC.Type, re.IGNORECASE):
+					Target = EPC.MmeIPAddress
+				else:
+					Target = EPC.IPAddress
+				#ping from module NIC rather than IP address to make sure round trip is over the air	
+				interface = f'-I {ue.getIFName()}' if ue.getIFName() else ''
+				ping_cmd = f'{ue.getCmdPrefix()} ping {interface} {self.ping_args} {Target} &> /tmp/{ping_log_file}'
+				cmd = cls_cmd.getConnection(ue.getHost())
+				response = cmd.run(ping_cmd, timeout=int(ping_time[0])*1.5)
+				if response.returncode != 0:
+					ping_status = -1
+				else:
 					#copy the ping log file to have it locally for analysis (ping stats)
-					SSH.copyin(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword, 'ping_' + self.testCase_id + '_' + self.ue_id + '.log', '.')
-
-					#cat is executed locally 
-					SSH.command('cat ping_' + self.testCase_id + '_' + self.ue_id + '.log', '\$', 5)
-					ping_log_file='ping_' + self.testCase_id + '_' + self.ue_id + '.log'
-
-				elif (launchFromASUE == True): 
-					#ping was already executed when running scenario
-					#we only need to retrieve ping log file, whose location is in the ci_ueinfra.yaml
-					logging.debug("Get logs from AS server : " + Module_UE.Ping + ", " + Module_UE.UELog)
-					SSH.copyin(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword, Module_UE.Ping, '.')
-					SSH.copyin(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword, Module_UE.UELog, '.')
-					logging.debug("Ping analysis from Amarisoft scenario")
-					path,ping_log_file = os.path.split(Module_UE.Ping)
-					SSH.open(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword)
-					SSH.command('cat ' + Module_UE.Ping, '\$', 5)
+					cmd.copyin(src=f'/tmp/{ping_log_file}', tgt=ping_log_file)
 
-				else:
-					ping_status=-1
+				cmd.close()
 
 			# TIMEOUT CASE
 			if ping_status < 0:
-				message = 'Ping with UE (' + str(UE_IPAddress) + ') crashed due to TIMEOUT!'
+				message = 'Ping with UE (' + str(ue.getIP()) + ') crashed due to TIMEOUT!'
 				logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m')
-				SSH.close()
-				self.ping_iperf_wrong_exit(lock, UE_IPAddress, device_id, statusQueue, message)
+				self.ping_iperf_wrong_exit(lock, ue.getIP(), ue.getName(), statusQueue, message)
 				return
 			#search is done on cat result
-			result = re.search(', (?P<packetloss>[0-9\.]+)% packet loss, time [0-9\.]+ms', SSH.getBefore())
+			with open(ping_log_file, 'r') as f:
+				ping_output = "".join(f.readlines())
+			result = re.search(', (?P<packetloss>[0-9\.]+)% packet loss, time [0-9\.]+ms', ping_output)
 			if result is None:
 				message = 'Packet Loss Not Found!'
 				logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m')
-				SSH.close()
-				self.ping_iperf_wrong_exit(lock, UE_IPAddress, device_id, statusQueue, message)
+				self.ping_iperf_wrong_exit(lock, ue.getIP(), ue.getName(), statusQueue, message)
 				return
 			packetloss = result.group('packetloss')
 			if float(packetloss) == 100:
 				message = 'Packet Loss is 100%'
 				logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m')
-				SSH.close()
-				self.ping_iperf_wrong_exit(lock, UE_IPAddress, device_id, statusQueue, message)
+				self.ping_iperf_wrong_exit(lock, ue.getIP(), ue.getName(), statusQueue, message)
 				return
-			result = re.search('rtt min\/avg\/max\/mdev = (?P<rtt_min>[0-9\.]+)\/(?P<rtt_avg>[0-9\.]+)\/(?P<rtt_max>[0-9\.]+)\/[0-9\.]+ ms', SSH.getBefore())
+			result = re.search('rtt min\/avg\/max\/mdev = (?P<rtt_min>[0-9\.]+)\/(?P<rtt_avg>[0-9\.]+)\/(?P<rtt_max>[0-9\.]+)\/[0-9\.]+ ms', ping_output)
 			if result is None:
 				message = 'Ping RTT_Min RTT_Avg RTT_Max Not Found!'
 				logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m')
-				SSH.close()
-				self.ping_iperf_wrong_exit(lock, UE_IPAddress, device_id, statusQueue, message)
+				self.ping_iperf_wrong_exit(lock, ue.getIP(), ue.getName(), statusQueue, message)
 				return
 			rtt_min = result.group('rtt_min')
 			rtt_avg = result.group('rtt_avg')
@@ -873,7 +828,7 @@ class OaiCiTest():
 			max_msg = 'RTT(Max)    : ' + rtt_max + ' ms'
 
 			lock.acquire()
-			logging.debug('\u001B[1;37;44m ping result (' + UE_IPAddress + ') \u001B[0m')
+			logging.debug('\u001B[1;37;44m ping result (' + ue.getIP() + ') \u001B[0m')
 			logging.debug('\u001B[1;34m    ' + pal_msg + '\u001B[0m')
 			logging.debug('\u001B[1;34m    ' + min_msg + '\u001B[0m')
 			logging.debug('\u001B[1;34m    ' + avg_msg + '\u001B[0m')
@@ -882,22 +837,21 @@ class OaiCiTest():
 			#adding extra ping stats from local file
 			#ping_log_file variable is defined above in this function, depending on device/ue
 			ping_stat_msg=''
-			if launchFromASUE == False : #skip in case of AS UE (for the moment)
-				logging.debug('Analyzing Ping log file : ' + os.getcwd() + '/' + ping_log_file)
-				ping_stat=GetPingTimeAnalysis(RAN,ping_log_file,self.ping_rttavg_threshold)
-
-				if (ping_stat!=-1) and (len(ping_stat)!=0):
-					ping_stat_msg+='Ping stats before removing largest value : \n'
-					ping_stat_msg+='RTT(Min)    : ' + str("{:.2f}".format(ping_stat['min_0'])) + 'ms \n'
-					ping_stat_msg+='RTT(Mean)   : ' + str("{:.2f}".format(ping_stat['mean_0'])) + 'ms \n'
-					ping_stat_msg+='RTT(Median) : ' + str("{:.2f}".format(ping_stat['median_0'])) + 'ms \n'
-					ping_stat_msg+='RTT(Max)    : ' + str("{:.2f}".format(ping_stat['max_0'])) + 'ms \n'
-					ping_stat_msg+='Max Index   : ' + str(ping_stat['max_loc']) + '\n'
-					ping_stat_msg+='Ping stats after removing largest value : \n'
-					ping_stat_msg+='RTT(Min)    : ' + str("{:.2f}".format(ping_stat['min_1'])) + 'ms \n'
-					ping_stat_msg+='RTT(Mean)   : ' + str("{:.2f}".format(ping_stat['mean_1'])) + 'ms \n'
-					ping_stat_msg+='RTT(Median) : ' + str("{:.2f}".format(ping_stat['median_1'])) + 'ms \n'
-					ping_stat_msg+='RTT(Max)    : ' + str("{:.2f}".format(ping_stat['max_1'])) + 'ms \n'
+			logging.debug('Analyzing Ping log file : ' + os.getcwd() + '/' + ping_log_file)
+			ping_stat=GetPingTimeAnalysis(RAN,ping_log_file,self.ping_rttavg_threshold)
+
+			if (ping_stat!=-1) and (len(ping_stat)!=0):
+				ping_stat_msg+='Ping stats before removing largest value : \n'
+				ping_stat_msg+='RTT(Min)    : ' + str("{:.2f}".format(ping_stat['min_0'])) + 'ms \n'
+				ping_stat_msg+='RTT(Mean)   : ' + str("{:.2f}".format(ping_stat['mean_0'])) + 'ms \n'
+				ping_stat_msg+='RTT(Median) : ' + str("{:.2f}".format(ping_stat['median_0'])) + 'ms \n'
+				ping_stat_msg+='RTT(Max)    : ' + str("{:.2f}".format(ping_stat['max_0'])) + 'ms \n'
+				ping_stat_msg+='Max Index   : ' + str(ping_stat['max_loc']) + '\n'
+				ping_stat_msg+='Ping stats after removing largest value : \n'
+				ping_stat_msg+='RTT(Min)    : ' + str("{:.2f}".format(ping_stat['min_1'])) + 'ms \n'
+				ping_stat_msg+='RTT(Mean)   : ' + str("{:.2f}".format(ping_stat['mean_1'])) + 'ms \n'
+				ping_stat_msg+='RTT(Median) : ' + str("{:.2f}".format(ping_stat['median_1'])) + 'ms \n'
+				ping_stat_msg+='RTT(Max)    : ' + str("{:.2f}".format(ping_stat['max_1'])) + 'ms \n'
 
 			#building html message
 			qMsg = pal_msg + '\n' + min_msg + '\n' + avg_msg + '\n' + max_msg + '\n'  + ping_stat_msg
@@ -926,11 +880,10 @@ class OaiCiTest():
 				statusQueue.put(0)
 			else:
 				statusQueue.put(-1)
-			statusQueue.put(device_id)
-			statusQueue.put(UE_IPAddress)
+			statusQueue.put(ue.getName())
+			statusQueue.put(ue.getIP())
 			statusQueue.put(qMsg)
 			lock.release()
-			SSH.close()
 		except:
 			logging.debug('exit from Ping_Common except')
 			os.kill(os.getppid(),signal.SIGUSR1)
@@ -1049,31 +1002,23 @@ class OaiCiTest():
 			return
 
 		if self.ue_id == "":
-			logging.error("no module ID!!!")
-			exit(1)
-
-		self.UEIPAddresses = []
-		ue = cls_module_ue.Module_UE(self.ue_id)
-		ip = ue.getIP()
-		if ip is None:
-			logging.error("no IP addresses returned")
-			html_queue = SimpleQueue()
-			HTML.CreateHtmlTestRowQueue(self.ping_args, 'KO', len(self.UEDevices), html_queue)
-			self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,CONTAINERS)
-		self.UEIPAddresses.append(ip)
+			raise Exception("no module names in self.ue_id provided")
+		ues = []
+		for ue_name in self.ue_id.split(' '):
+			ue = cls_module_ue.Module_UE(ue_name.strip())
+			if not ue.getIP():
+				logging.error("no IP addresses returned")
+				HTML.CreateHtmlTestRow(self.ping_args, 'KO', len(self.UEDevices), html_queue)
+				self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,CONTAINERS)
+			ues.append(ue)
+		logging.debug(ues)
 
-		logging.debug(self.UEIPAddresses)
 		multi_jobs = []
 		i = 0
 		lock = Lock()
 		status_queue = SimpleQueue()
-		for UE_IPAddress in self.UEIPAddresses:
-			if self.ue_id=="":
-				device_id = self.UEDevices[i]
-			else:
-				device_id = Module_UE.ID + "-" + Module_UE.Kind 
-				logging.debug(device_id)
-			p = Process(target = self.Ping_common, args = (lock,UE_IPAddress,device_id,status_queue,EPC,Module_UE,RAN,))
+		for ue in ues:
+			p = Process(target = self.Ping_common, args = (lock,status_queue,EPC,ue,RAN,))
 			p.daemon = True
 			p.start()
 			multi_jobs.append(p)