Skip to content

O
OpenXG-RAN
Commit 93f15bc9 authored by Remi Hardy's avatar Remi Hardy
Browse files
Options

Merge branch 'ci_improve_iperf_pass_fail' into 'develop' 

[CI] improve iperf pass fail

See merge request oai/openairinterface5g!1373
parents 8d1c9894 efcc3f55
Hide whitespace changes
Inline Side-by-side
Showing with 121 additions and 19 deletions
ci-scripts/cls_oaicitest.py
View file @ 93f15bc9
......@@ -57,12 +57,15 @@ import sshconnection
import cls_module_ue
import cls_ci_ueinfra #class defining the multi Ue infrastrucure
logging.getLogger("matplotlib").setLevel(logging.WARNING)
import matplotlib.pyplot as plt
import numpy as np
#-----------------------------------------------------------
# Utility functions
#-----------------------------------------------------------
def GetPingTimeAnalysis(ping_log_file):
def GetPingTimeAnalysis(RAN,ping_log_file,ping_rttavg_threshold):
#ping time values read from file
t_ping=[]
#ping stats (dictionary) to be returned by the function
......@@ -86,17 +89,46 @@ def GetPingTimeAnalysis(ping_log_file):
max_loc=t_ping.index(max(t_ping))
ping_stat['max_loc']=max_loc
#remove it
t_ping.pop(max_loc)
t_ping_post=t_ping.copy()
t_ping_post.pop(max_loc)
#new stats after removing max value
ping_stat['min_1']=min(t_ping)
ping_stat['mean_1']=stat.mean(t_ping)
ping_stat['median_1']=stat.median(t_ping)
ping_stat['max_1']=max(t_ping)
ping_stat['min_1']=min(t_ping_post)
ping_stat['mean_1']=stat.mean(t_ping_post)
ping_stat['median_1']=stat.median(t_ping_post)
ping_stat['max_1']=max(t_ping_post)
#plot ping over time and save png for artifacts
ticks = np.arange(0, len(t_ping), 1)
figure, axis = plt.subplots(figsize=(10, 10))
axis.plot(ticks,t_ping,marker='o')
axis.set_xlabel('Ping Events')
axis.set_ylabel("Ping RTT (in ms)")
axis.set_title(ping_log_file)
axis.set_xticks(ticks)
axis.set_xticklabels([])
YMAX=20 #base scale
if max(t_ping) > YMAX:
y_max=max(t_ping)+1
else:
y_max=YMAX+1
plt.ylim(0,y_max)
if ping_rttavg_threshold != '':
th_label="AVG Ping Fail Threshold="+ping_rttavg_threshold
plt.axhline(y=float(ping_rttavg_threshold), color='r', linestyle='-',label=th_label)
axis.legend()
plt.savefig(ping_log_file+'.png')
#copy the png file already to enb to move it move it later into the artifacts
try:
mySSH = sshconnection.SSHConnection()
mySSH.copyout(RAN.eNBIPAddress, RAN.eNBUserName, RAN.eNBPassword, ping_log_file+'.png', RAN.eNBSourceCodePath + '/cmake_targets/')
except:
logging.debug('\u001B[1;37;41m Ping PNG SCP to eNB FAILED\u001B[0m')
return ping_stat
else:
logging.error("Ping log file does not exist")
logging.error("GetPingTimeAnalysis : Ping log file does not exist")
return -1
......@@ -129,6 +161,7 @@ class OaiCiTest():
self.ping_rttavg_threshold =''
self.iperf_args = ''
self.iperf_packetloss_threshold = ''
self.iperf_bitrate_threshold = ''
self.iperf_profile = ''
self.iperf_options = ''
self.iperf_direction = ''
......@@ -1531,7 +1564,7 @@ class OaiCiTest():
statusQueue.put(message)
lock.release()
def Ping_common(self, lock, UE_IPAddress, device_id, statusQueue,EPC, Module_UE):
def Ping_common(self, lock, UE_IPAddress, device_id, statusQueue,EPC, Module_UE,RAN):
try:
SSH = sshconnection.SSHConnection()
# Launch ping on the EPC side (true for ltebox and old open-air-cn)
......@@ -1559,6 +1592,7 @@ class OaiCiTest():
ping_status = SSH.command('docker exec -it prod-trf-gen /bin/bash -c "ping ' + self.ping_args + ' ' + UE_IPAddress + '" 2>&1 | tee ping_' + self.testCase_id + '_' + device_id + '.log', '\$', int(ping_time[0])*1.5)
else:
ping_status = SSH.command('stdbuf -o0 ping ' + self.ping_args + ' ' + UE_IPAddress + ' 2>&1 | stdbuf -o0 tee ping_' + self.testCase_id + '_' + device_id + '.log', '\$', int(ping_time[0])*1.5)
ping_log_file='ping_' + self.testCase_id + '_' + device_id + '.log'
#copy the ping log file to have it locally for analysis (ping stats)
SSH.copyin(EPC.IPAddress, EPC.UserName, EPC.Password, EPC.SourceCodePath + '/scripts/ping_' + self.testCase_id + '_' + device_id + '.log', '.')
else:
......@@ -1578,6 +1612,7 @@ class OaiCiTest():
SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password)
#cat is executed on EPC
SSH.command('cat ' + EPC.SourceCodePath + '/scripts/ping_' + self.testCase_id + '_' + device_id + '.log', '\$', 5)
ping_log_file='/scripts/ping_' + self.testCase_id + '_' + device_id + '.log'
else: #launch from Module
SSH.open(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword)
#target address is different depending on EPC type
......@@ -1595,6 +1630,7 @@ class OaiCiTest():
#cat is executed locally
SSH.command('cat ping_' + self.testCase_id + '_' + self.ue_id + '.log', '\$', 5)
ping_log_file='ping_' + self.testCase_id + '_' + self.ue_id + '.log'
ping_status=0
# TIMEOUT CASE
......@@ -1642,9 +1678,9 @@ class OaiCiTest():
logging.debug('\u001B[1;34m ' + max_msg + '\u001B[0m')
#adding extra ping stats from local file
ping_log_file='ping_' + self.testCase_id + '_' + device_id + '.log'
#ping_log_file variable is defined above in this function, depending on device/ue
logging.debug('Analyzing Ping log file : ' + os.getcwd() + '/' + ping_log_file)
ping_stat=GetPingTimeAnalysis(ping_log_file)
ping_stat=GetPingTimeAnalysis(RAN,ping_log_file,self.ping_rttavg_threshold)
ping_stat_msg=''
if (ping_stat!=-1) and (len(ping_stat)!=0):
ping_stat_msg+='Ping stats before removing largest value : \n'
......@@ -1834,7 +1870,7 @@ class OaiCiTest():
device_id = self.UEDevices[i]
else:
device_id = Module_UE.ID + "-" + Module_UE.Kind
p = Process(target = self.Ping_common, args = (lock,UE_IPAddress,device_id,status_queue,EPC,Module_UE,))
p = Process(target = self.Ping_common, args = (lock,UE_IPAddress,device_id,status_queue,EPC,Module_UE,RAN,))
p.daemon = True
p.start()
multi_jobs.append(p)
......@@ -2085,12 +2121,20 @@ class OaiCiTest():
pl = float(100 * pl_sum / ps_sum)
packetloss = '%2.1f ' % (pl)
packetloss += '%'
#checking packet loss compliance
if float(pl) > float(self.iperf_packetloss_threshold):
pal_too_high_msg = 'Packet Loss too high : actual = '+packetloss+', target = '+self.iperf_packetloss_threshold+'%\n'
pal_too_high_msg = 'Packet Loss too high : tested = '+packetloss+', target = '+self.iperf_packetloss_threshold+'%'
else:
pal_too_high_msg=''
pal_too_high_msg='Packet Loss value is within acceptance range'
#checking bitrate perf compliance
if float(br_loss) < float(self.iperf_bitrate_threshold):
bit_too_low_msg = 'Bitrate too low : tested = '+bitperf+', target = '+self.iperf_bitrate_threshold+'%'
else:
bit_too_low_msg='Bitrate perf value is within acceptance range'
lock.acquire()
if (br_loss < 90) or (float(pl) > float(self.iperf_packetloss_threshold)):
if (float(br_loss) < float(self.iperf_bitrate_threshold)) and (float(pl) > float(self.iperf_packetloss_threshold)):
statusQueue.put(-1)
elif (float(br_loss) < float(self.iperf_bitrate_threshold)) or (float(pl) > float(self.iperf_packetloss_threshold)):
statusQueue.put(1)
else:
statusQueue.put(0)
......@@ -2101,7 +2145,7 @@ class OaiCiTest():
brl_msg = 'Bitrate Perf: ' + bitperf
jit_msg = 'Jitter : ' + jitter
pal_msg = 'Packet Loss : ' + packetloss
statusQueue.put(req_msg + '\n' + bir_msg + '\n' + brl_msg + '\n' + jit_msg + '\n' + pal_msg + '\n' + pal_too_high_msg + '\n')
statusQueue.put(req_msg + '\n' + bir_msg + '\n' + brl_msg + '\n' + jit_msg + '\n' + pal_msg + '\n' + pal_too_high_msg + '\n' + bit_too_low_msg + '\n')
logging.debug('\u001B[1;37;45m iperf result (' + UE_IPAddress + ') \u001B[0m')
logging.debug('\u001B[1;35m ' + req_msg + '\u001B[0m')
logging.debug('\u001B[1;35m ' + bir_msg + '\u001B[0m')
......@@ -2109,6 +2153,7 @@ class OaiCiTest():
logging.debug('\u001B[1;35m ' + jit_msg + '\u001B[0m')
logging.debug('\u001B[1;35m ' + pal_msg + '\u001B[0m')
logging.debug('\u001B[1;35m ' + pal_too_high_msg + '\u001B[0m')
logging.debug('\u001B[1;35m ' + bit_too_low_msg + '\u001B[0m')
lock.release()
else:
self.ping_iperf_wrong_exit(lock, UE_IPAddress, device_id, statusQueue, 'Could not analyze from server log')
......@@ -3391,6 +3436,41 @@ class OaiCiTest():
CONTAINERS.UndeployObject(HTML,RAN)
RAN.prematureExit=True
#this function is called only if eNB/gNB fails to start
#RH to be re-factored
def AutoTerminateeNB(self,HTML,RAN,EPC,CONTAINERS):
if (RAN.Initialize_eNB_args != ''):
self.testCase_id = 'AUTO-KILL-RAN'
HTML.testCase_id = self.testCase_id
self.desc = 'Automatic Termination of all RAN nodes'
HTML.desc = self.desc
self.ShowTestID()
#terminate all RAN nodes eNB/gNB/OCP
for instance in range(0, len(RAN.air_interface)):
if RAN.air_interface[instance]!='':
logging.debug('Auto Termination of Instance ' + str(instance) + ' : ' + RAN.air_interface[instance])
RAN.eNB_instance=instance
RAN.TerminateeNB(HTML,EPC)
if RAN.flexranCtrlInstalled and RAN.flexranCtrlStarted:
self.testCase_id = 'AUTO-KILL-flexran-ctl'
HTML.testCase_id = self.testCase_id
self.desc = 'Automatic Termination of FlexRan CTL'
HTML.desc = self.desc
self.ShowTestID()
self.TerminateFlexranCtrl(HTML,RAN,EPC)
if CONTAINERS.yamlPath[0] != '':
self.testCase_id = 'AUTO-KILL-CONTAINERS'
HTML.testCase_id = self.testCase_id
self.desc = 'Automatic Termination of all RAN containers'
HTML.desc = self.desc
self.ShowTestID()
for instance in range(0, len(CONTAINERS.yamlPath)):
if CONTAINERS.yamlPath[instance]!='':
CONTAINERS.eNB_instance=instance
CONTAINERS.UndeployObject(HTML,RAN)
RAN.prematureExit=True
def IdleSleep(self,HTML):
time.sleep(self.idle_sleep_time)
HTML.CreateHtmlTestRow(str(self.idle_sleep_time) + ' sec', 'OK', CONST.ALL_PROCESSES_OK)
......
ci-scripts/conf_files/enb.band38.lte_2x2.100PRB.usrpn310.conf
View file @ 93f15bc9
......@@ -237,7 +237,7 @@ RUs = (
max_pdschReferenceSignalPower = -27;
max_rxgain = 75;
eNB_instances = [0];
sdr_addrs = "mgmt_addr=192.168.18.241,addr=192.168.10.2";
sdr_addrs = "mgmt_addr=192.168.18.241,addr=192.168.20.2,second_addr=192.168.10.2";
}
);
......
ci-scripts/conf_files/enb.band38.nsa_2x2.100PRB.usrpn310.conf
View file @ 93f15bc9
......@@ -237,7 +237,7 @@ RUs = (
max_pdschReferenceSignalPower = -27;
max_rxgain = 75;
eNB_instances = [0];
sdr_addrs = "mgmt_addr=192.168.18.241,addr=192.168.10.2";
sdr_addrs = "mgmt_addr=192.168.18.241,addr=192.168.20.2,second_addr=192.168.10.2";
}
);
......
ci-scripts/main.py
View file @ 93f15bc9
......@@ -310,6 +310,13 @@ def GetParametersFromXML(action):
CiTestObj.ue_id = ue_id
CiTestObj.iperf_direction = test.findtext('direction')#used for modules only
CiTestObj.iperf_packetloss_threshold = test.findtext('iperf_packetloss_threshold')
iperf_bitrate_threshold = test.findtext('iperf_bitrate_threshold')
if (iperf_bitrate_threshold is None):
CiTestObj.iperf_bitrate_threshold = "90" #if no threshold is specified, default will be 90%
else:
CiTestObj.iperf_bitrate_threshold = iperf_bitrate_threshold
CiTestObj.iperf_profile = test.findtext('iperf_profile')
if (CiTestObj.iperf_profile is None):
CiTestObj.iperf_profile = 'balanced'
......@@ -811,6 +818,8 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re
check_OAI_UE = False
RAN.pStatus=CiTestObj.CheckProcessExist(check_eNB, check_OAI_UE,RAN,EPC)
RAN.InitializeeNB(HTML, EPC)
if RAN.prematureExit:
CiTestObj.AutoTerminateeNB(HTML,RAN,EPC,CONTAINERS)
elif action == 'Terminate_eNB':
RAN.TerminateeNB(HTML, EPC)
elif action == 'Initialize_UE':
......
ci-scripts/ran.py
View file @ 93f15bc9
......@@ -738,8 +738,8 @@ class RANManagement():
mySSH.command('echo ' + self.eNBPassword + ' | sudo -S mv /tmp/enb_*.pcap .','\$',20)
mySSH.command('echo ' + self.eNBPassword + ' | sudo -S mv /tmp/gnb_*.pcap .','\$',20)
mySSH.command('echo ' + self.eNBPassword + ' | sudo -S rm -f enb.log.zip', '\$', 5)
mySSH.command('echo ' + self.eNBPassword + ' | sudo -S zip enb.log.zip enb*.log core* enb_*record.raw enb_*.pcap gnb_*.pcap enb_*txt physim_*.log *stats.log *monitor.pickle *monitor*.png log/*/*.log log/*/*.pcap', '\$', 60)
mySSH.command('echo ' + self.eNBPassword + ' | sudo -S rm enb*.log core* enb_*record.raw enb_*.pcap gnb_*.pcap enb_*txt physim_*.log *stats.log *monitor.pickle *monitor*.png log/*/*.log log/*/*.pcap', '\$', 15)
mySSH.command('echo ' + self.eNBPassword + ' | sudo -S zip enb.log.zip enb*.log core* enb_*record.raw enb_*.pcap gnb_*.pcap enb_*txt physim_*.log *stats.log *monitor.pickle *monitor*.png ping*.log.png log/*/*.log log/*/*.pcap', '\$', 60)
mySSH.command('echo ' + self.eNBPassword + ' | sudo -S rm enb*.log core* enb_*record.raw enb_*.pcap gnb_*.pcap enb_*txt physim_*.log *stats.log *monitor.pickle *monitor*.png ping*.log.png log/*/*.log log/*/*.pcap', '\$', 15)
mySSH.close()
def AnalyzeLogFile_eNB(self, eNBlogFile, HTML):
......
ci-scripts/xml_files/container_nsa_b200_quectel.xml
View file @ 93f15bc9
......@@ -124,6 +124,7 @@
<direction>DL</direction>
<id>idefix</id>
<iperf_packetloss_threshold>3</iperf_packetloss_threshold>
<iperf_bitrate_threshold>95</iperf_bitrate_threshold>
<iperf_profile>single-ue</iperf_profile>
</testCase>
......@@ -134,6 +135,7 @@
<direction>UL</direction>
<id>idefix</id>
<iperf_packetloss_threshold>1</iperf_packetloss_threshold>
<iperf_bitrate_threshold>95</iperf_bitrate_threshold>
<iperf_profile>single-ue</iperf_profile>
</testCase>
......
ci-scripts/xml_files/fr1_lte_2x2_quectel.xml
View file @ 93f15bc9
......@@ -107,6 +107,7 @@
<direction>DL</direction>
<id>nrmodule2_quectel</id>
<iperf_packetloss_threshold>5</iperf_packetloss_threshold>
<iperf_bitrate_threshold>95</iperf_bitrate_threshold>
<iperf_profile>single-ue</iperf_profile>
</testCase>
......@@ -117,6 +118,7 @@
<direction>UL</direction>
<id>nrmodule2_quectel</id>
<iperf_packetloss_threshold>5</iperf_packetloss_threshold>
<iperf_bitrate_threshold>95</iperf_bitrate_threshold>
<iperf_profile>single-ue</iperf_profile>
</testCase>
......
ci-scripts/xml_files/fr1_nsa_quectel.xml
View file @ 93f15bc9
......@@ -119,6 +119,7 @@
<direction>DL</direction>
<id>idefix</id>
<iperf_packetloss_threshold>3</iperf_packetloss_threshold>
<iperf_bitrate_threshold>95</iperf_bitrate_threshold>
<iperf_profile>single-ue</iperf_profile>
</testCase>
......@@ -129,6 +130,7 @@
<direction>UL</direction>
<id>idefix</id>
<iperf_packetloss_threshold>1</iperf_packetloss_threshold>
<iperf_bitrate_threshold>95</iperf_bitrate_threshold>
<iperf_profile>single-ue</iperf_profile>
</testCase>
......
ci-scripts/xml_files/fr1_sa_quectel.xml
View file @ 93f15bc9
......@@ -105,6 +105,7 @@
<direction>DL</direction>
<id>nrmodule2_quectel</id>
<iperf_packetloss_threshold>5</iperf_packetloss_threshold>
<iperf_bitrate_threshold>95</iperf_bitrate_threshold>
<iperf_profile>single-ue</iperf_profile>
</testCase>
......@@ -115,6 +116,7 @@
<direction>UL</direction>
<id>nrmodule2_quectel</id>
<iperf_packetloss_threshold>5</iperf_packetloss_threshold>
<iperf_bitrate_threshold>95</iperf_bitrate_threshold>
<iperf_profile>single-ue</iperf_profile>
</testCase>
......
ci-scripts/xml_files/fr1_sa_quectel_stages.xml
View file @ 93f15bc9
......@@ -104,6 +104,7 @@
<direction>DL</direction>
<id>nrmodule2_quectel</id>
<iperf_packetloss_threshold>5</iperf_packetloss_threshold>
<iperf_bitrate_threshold>95</iperf_bitrate_threshold>
<iperf_profile>single-ue</iperf_profile>
</testCase>
<testCase id="070001">
......@@ -113,6 +114,7 @@
<direction>DL</direction>
<id>nrmodule2_quectel</id>
<iperf_packetloss_threshold>5</iperf_packetloss_threshold>
<iperf_bitrate_threshold>95</iperf_bitrate_threshold>
<iperf_profile>single-ue</iperf_profile>
</testCase>
<testCase id="070002">
......@@ -122,6 +124,7 @@
<direction>DL</direction>
<id>nrmodule2_quectel</id>
<iperf_packetloss_threshold>5</iperf_packetloss_threshold>
<iperf_bitrate_threshold>95</iperf_bitrate_threshold>
<iperf_profile>single-ue</iperf_profile>
</testCase>
<testCase id="070003">
......@@ -131,6 +134,7 @@
<direction>DL</direction>
<id>nrmodule2_quectel</id>
<iperf_packetloss_threshold>5</iperf_packetloss_threshold>
<iperf_bitrate_threshold>95</iperf_bitrate_threshold>
<iperf_profile>single-ue</iperf_profile>
</testCase>
<testCase id="070004">
......@@ -140,6 +144,7 @@
<direction>DL</direction>
<id>nrmodule2_quectel</id>
<iperf_packetloss_threshold>5</iperf_packetloss_threshold>
<iperf_bitrate_threshold>95</iperf_bitrate_threshold>
<iperf_profile>single-ue</iperf_profile>
</testCase>
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment