Commit 23a804ec authored by Raphael Defosseux's avatar Raphael Defosseux

Merge branch 'ci-repeat-scenario' into 'develop'

CI: adding support to repeat scenario in case of failure.

See merge request oai/openairinterface5g!645
parents 2d6d1dd9 65002b7f
......@@ -87,6 +87,7 @@ logging.basicConfig(
#-----------------------------------------------------------
class SSHConnection():
def __init__(self):
self.prematureExit = False
self.ranRepository = ''
self.ranBranch = ''
self.ranAllowMerge = False
......@@ -146,6 +147,7 @@ class SSHConnection():
self.htmlTabRefs = []
self.htmlTabNames = []
self.htmlTabIcons = []
self.repeatCounts = []
self.finalStatus = False
self.OsVersion = ''
self.KernelVersion = ''
......@@ -745,7 +747,6 @@ class SSHConnection():
doLoop = False
logging.error('\u001B[1;37;41m eNB logging system did not show got sync! \u001B[0m')
self.CreateHtmlTestRow('-O ' + config_file + extra_options, 'KO', ALL_PROCESSES_OK)
self.CreateHtmlTabFooter(False)
# In case of T tracer recording, we need to kill tshark on EPC side
result = re.search('T_stdout', str(self.Initialize_eNB_args))
if result is not None:
......@@ -761,7 +762,8 @@ class SSHConnection():
copyin_res = self.copyin(self.EPCIPAddress, self.EPCUserName, self.EPCPassword, '/tmp/' + self.EPC_PcapFileName, '.')
if (copyin_res == 0):
self.copyout(lIpAddr, lUserName, lPassWord, self.EPC_PcapFileName, lSourcePath + '/cmake_targets/.')
sys.exit(1)
self.prematureExit = True
return
else:
self.command('stdbuf -o0 cat enb_' + self.testCase_id + '.log | egrep --text --color=never -i "wait|sync|Starting"', '\$', 4)
if rruCheck:
......@@ -938,8 +940,6 @@ class SSHConnection():
self.CreateHtmlTestRow(self.Initialize_OAI_UE_args, 'KO', OAI_UE_PROCESS_NO_TUNNEL_INTERFACE, 'OAI UE')
logging.error('\033[91mInitialize OAI UE Failed! \033[0m')
self.AutoTerminateUEandeNB()
self.CreateHtmlTabFooter(False)
sys.exit(1)
def checkDevTTYisUnlocked(self):
self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
......@@ -1090,8 +1090,8 @@ class SSHConnection():
pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE)
if (pStatus < 0):
self.CreateHtmlTestRow(self.ping_args, 'KO', pStatus)
self.CreateHtmlTabFooter(False)
sys.exit(1)
self.prematureExit = True
return
try:
statusQueue = SimpleQueue()
lock = Lock()
......@@ -1173,8 +1173,6 @@ class SSHConnection():
else:
self.CreateHtmlTestRowQueue(self.ping_args, 'KO', 1, statusQueue)
self.AutoTerminateUEandeNB()
self.CreateHtmlTabFooter(False)
sys.exit(1)
except:
os.kill(os.getppid(),signal.SIGUSR1)
......@@ -1235,8 +1233,7 @@ class SSHConnection():
if (pStatus < 0):
self.CreateHtmlTestRow('N/A', 'KO', pStatus)
self.AutoTerminateUEandeNB()
self.CreateHtmlTabFooter(False)
sys.exit(1)
return
multi_jobs = []
status_queue = SimpleQueue()
lock = Lock()
......@@ -1254,9 +1251,8 @@ class SSHConnection():
if (status_queue.empty()):
self.CreateHtmlTestRow('N/A', 'KO', ALL_PROCESSES_OK)
self.CreateHtmlTabFooter(False)
self.AutoTerminateUEandeNB()
sys.exit(1)
return
else:
attach_status = True
html_queue = SimpleQueue()
......@@ -1285,8 +1281,6 @@ class SSHConnection():
else:
self.CreateHtmlTestRowQueue('N/A', 'KO', len(self.UEDevices), html_queue)
self.AutoTerminateUEandeNB()
self.CreateHtmlTabFooter(False)
sys.exit(1)
def DetachUE_common(self, device_id):
try:
......@@ -1307,8 +1301,7 @@ class SSHConnection():
if (pStatus < 0):
self.CreateHtmlTestRow('N/A', 'KO', pStatus)
self.AutoTerminateUEandeNB()
self.CreateHtmlTabFooter(False)
sys.exit(1)
return
multi_jobs = []
cnt = 0
for device_id in self.UEDevices:
......@@ -1560,8 +1553,6 @@ class SSHConnection():
if (status_queue.empty()):
self.CreateHtmlTestRow(htmlOptions, 'KO', ALL_PROCESSES_OK)
self.AutoTerminateUEandeNB()
self.CreateHtmlTabFooter(False)
sys.exit(1)
else:
check_status = True
html_queue = SimpleQueue()
......@@ -1578,8 +1569,6 @@ class SSHConnection():
else:
self.CreateHtmlTestRowQueue(htmlOptions, 'KO', len(self.UEDevices), html_queue)
self.AutoTerminateUEandeNB()
self.CreateHtmlTabFooter(False)
sys.exit(1)
def GetAllUEIPAddresses(self):
if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '':
......@@ -1725,8 +1714,7 @@ class SSHConnection():
if (pStatus < 0):
self.CreateHtmlTestRow(self.ping_args, 'KO', pStatus)
self.AutoTerminateUEandeNB()
self.CreateHtmlTabFooter(False)
sys.exit(1)
return
ping_from_eNB = re.search('oaitun_enb1', str(self.ping_args))
if ping_from_eNB is not None:
if self.eNBIPAddress == '' or self.eNBUserName == '' or self.eNBPassword == '':
......@@ -1829,14 +1817,12 @@ class SSHConnection():
if (pStatus < 0):
self.CreateHtmlTestRow(self.ping_args, 'KO', pStatus)
self.AutoTerminateUEandeNB()
self.CreateHtmlTabFooter(False)
sys.exit(1)
return
ueIpStatus = self.GetAllUEIPAddresses()
if (ueIpStatus < 0):
self.CreateHtmlTestRow(self.ping_args, 'KO', UE_IP_ADDRESS_ISSUE)
self.AutoTerminateUEandeNB()
self.CreateHtmlTabFooter(False)
sys.exit(1)
return
multi_jobs = []
i = 0
lock = Lock()
......@@ -1854,8 +1840,6 @@ class SSHConnection():
if (status_queue.empty()):
self.CreateHtmlTestRow(self.ping_args, 'KO', ALL_PROCESSES_OK)
self.AutoTerminateUEandeNB()
self.CreateHtmlTabFooter(False)
sys.exit(1)
else:
ping_status = True
html_queue = SimpleQueue()
......@@ -1873,8 +1857,6 @@ class SSHConnection():
else:
self.CreateHtmlTestRowQueue(self.ping_args, 'KO', len(self.UEDevices), html_queue)
self.AutoTerminateUEandeNB()
self.CreateHtmlTabFooter(False)
sys.exit(1)
def Iperf_ComputeTime(self):
result = re.search('-t (?P<iperf_time>\d+)', str(self.iperf_args))
......@@ -2352,8 +2334,6 @@ class SSHConnection():
if (pStatus < 0):
self.CreateHtmlTestRow(self.iperf_args, 'KO', pStatus)
self.AutoTerminateUEandeNB()
self.CreateHtmlTabFooter(False)
sys.exit(1)
if self.eNBIPAddress == '' or self.eNBUserName == '' or self.eNBPassword == '' or self.UEIPAddress == '' or self.UEUserName == '' or self.UEPassword == '':
Usage()
sys.exit('Insufficient Parameter')
......@@ -2443,8 +2423,6 @@ class SSHConnection():
else:
self.CreateHtmlTestRowQueue(self.iperf_args, 'KO', len(self.UEDevices), html_queue)
self.AutoTerminateUEandeNB()
self.CreateHtmlTabFooter(False)
sys.exit(1)
def Iperf(self):
result = re.search('noS1', str(self.Initialize_eNB_args))
......@@ -2463,14 +2441,12 @@ class SSHConnection():
if (pStatus < 0):
self.CreateHtmlTestRow(self.iperf_args, 'KO', pStatus)
self.AutoTerminateUEandeNB()
self.CreateHtmlTabFooter(False)
sys.exit(1)
return
ueIpStatus = self.GetAllUEIPAddresses()
if (ueIpStatus < 0):
self.CreateHtmlTestRow(self.iperf_args, 'KO', UE_IP_ADDRESS_ISSUE)
self.AutoTerminateUEandeNB()
self.CreateHtmlTabFooter(False)
sys.exit(1)
return
multi_jobs = []
i = 0
ue_num = len(self.UEIPAddresses)
......@@ -2489,8 +2465,6 @@ class SSHConnection():
if (status_queue.empty()):
self.CreateHtmlTestRow(self.iperf_args, 'KO', ALL_PROCESSES_OK)
self.AutoTerminateUEandeNB()
self.CreateHtmlTabFooter(False)
sys.exit(1)
else:
iperf_status = True
iperf_noperf = False
......@@ -2513,8 +2487,6 @@ class SSHConnection():
else:
self.CreateHtmlTestRowQueue(self.iperf_args, 'KO', len(self.UEDevices), html_queue)
self.AutoTerminateUEandeNB()
self.CreateHtmlTabFooter(False)
sys.exit(1)
def CheckProcessExist(self, check_eNB, check_OAI_UE):
multi_jobs = []
......@@ -2834,10 +2806,17 @@ class SSHConnection():
rrcMsg = ' -- ' + str(rrcReestablishReject) + ' were rejected'
logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
self.htmleNBFailureMsg += rrcMsg + '\n'
if self.eNBOptions[int(self.eNB_instance)] != '':
res1 = re.search('drx_Config_present prSetup', self.eNBOptions[int(self.eNB_instance)])
if res1 is not None:
if cdrxActivationMessageCount > 0:
rrcMsg = 'eNB activated the CDRX Configuration for ' + str(cdrxActivationMessageCount) + ' time(s)'
logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
self.htmleNBFailureMsg += rrcMsg + '\n'
else:
rrcMsg = 'eNB did NOT ACTIVATE the CDRX Configuration'
logging.debug('\u001B[1;37;43m ' + rrcMsg + ' \u001B[0m')
self.htmleNBFailureMsg += rrcMsg + '\n'
if rachCanceledProcedure > 0:
rachMsg = 'eNB cancelled ' + str(rachCanceledProcedure) + ' RA procedure(s)'
logging.debug('\u001B[1;30;43m ' + rachMsg + ' \u001B[0m')
......@@ -2852,6 +2831,7 @@ class SSHConnection():
rruMsg = 'Slave RRU DID NOT receive the RRU_frame_resynch command from RAU'
logging.debug('\u001B[1;37;41m ' + rruMsg + ' \u001B[0m')
self.htmleNBFailureMsg += rruMsg + '\n'
self.prematureExit = True
return ENB_PROCESS_SLAVE_RRU_NOT_SYNCED
if foundSegFault:
logging.debug('\u001B[1;37;41m eNB ended with a Segmentation Fault! \u001B[0m')
......@@ -3121,8 +3101,8 @@ class SSHConnection():
logStatus = self.AnalyzeLogFile_eNB(fileToAnalyze)
if (logStatus < 0):
self.CreateHtmlTestRow('N/A', 'KO', logStatus)
self.CreateHtmlTabFooter(False)
sys.exit(1)
self.preamtureExit = True
return
else:
self.CreateHtmlTestRow('N/A', 'OK', ALL_PROCESSES_OK)
else:
......@@ -3261,8 +3241,6 @@ class SSHConnection():
if (logStatus != OAI_UE_PROCESS_COULD_NOT_SYNC) or (ueAction != 'Sniffing'):
self.Initialize_OAI_UE_args = ''
self.AutoTerminateUEandeNB()
self.CreateHtmlTabFooter(False)
sys.exit(1)
else:
logging.debug('\u001B[1m' + ueAction + ' Completed \u001B[0m')
self.htmlUEFailureMsg = '<b>' + ueAction + ' Completed</b>\n' + self.htmlUEFailureMsg
......@@ -3288,6 +3266,7 @@ class SSHConnection():
self.ShowTestID()
self.eNB_instance = '0'
self.TerminateeNB()
self.prematureExit = True
def IdleSleep(self):
time.sleep(self.idle_sleep_time)
......@@ -3548,6 +3527,7 @@ class SSHConnection():
pillMsg = ' <li><a data-toggle="pill" href="#'
pillMsg += self.htmlTabRefs[count]
pillMsg += '">'
pillMsg += '__STATE_' + self.htmlTabNames[count] + '__'
pillMsg += self.htmlTabNames[count]
pillMsg += ' <span class="glyphicon glyphicon-'
pillMsg += self.htmlTabIcons[count]
......@@ -3601,6 +3581,14 @@ class SSHConnection():
self.htmlFile.write(' </tr>\n')
self.htmlFile.write(' </table>\n')
self.htmlFile.write(' </div>\n')
self.htmlFile.close()
time.sleep(1)
if passStatus:
cmd = "sed -i -e 's/__STATE_" + self.htmlTabNames[0] + "__//' test_results.html"
subprocess.run(cmd, shell=True)
else:
cmd = "sed -i -e 's/__STATE_" + self.htmlTabNames[0] + "__/<span class=\"glyphicon glyphicon-remove\"><\/span>/' test_results.html"
subprocess.run(cmd, shell=True)
self.htmlFooterCreated = False
def CreateHtmlFooter(self, passStatus):
......@@ -4178,6 +4166,9 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re
requested_tests=xmlRoot.findtext('TestCaseRequestedList',default='')
if (SSH.nbTestXMLfiles == 1):
SSH.htmlTabRefs.append(xmlRoot.findtext('htmlTabRef',default='test-tab-0'))
SSH.htmlTabNames.append(xmlRoot.findtext('htmlTabName',default='Test-0'))
repeatCount = xmlRoot.findtext('repeatCount',default='1')
SSH.repeatCounts.append(int(repeatCount))
all_tests=xmlRoot.findall('testCase')
exclusion_tests=exclusion_tests.split()
......@@ -4219,6 +4210,10 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re
SSH.CreateHtmlTabHeader()
cnt = 0
SSH.prematureExit = True
while cnt < SSH.repeatCounts[0] and SSH.prematureExit:
SSH.prematureExit = False
for test_case_id in todo_tests:
for test in all_tests:
id = test.get('id')
......@@ -4299,7 +4294,17 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re
SSH.IdleSleep()
else:
sys.exit('Invalid action')
if SSH.prematureExit:
break
if SSH.prematureExit:
break
cnt += 1
if cnt == SSH.repeatCounts[0] and SSH.prematureExit:
logging.debug('Testsuite failed ' + str(cnt) + ' time(s)')
SSH.CreateHtmlTabFooter(False)
sys.exit('Failed Scenario')
else:
logging.info('Testsuite passed after ' + str(cnt) + ' time(s)')
SSH.CreateHtmlTabFooter(True)
else:
Usage()
......
......@@ -28,7 +28,6 @@
030201
040101
030111 040301 040511 040613 040614 040615 040616 040617 040651 040652 040653 040654 040401 040201 030201
030112 040302 040512 040612 040650 040401 040201 030201
</TestCaseRequestedList>
<TestCaseExclusionList></TestCaseExclusionList>
......@@ -38,12 +37,6 @@
<Initialize_eNB_args>-O ci-scripts/conf_files/enb.band7.tm1.50PRB.usrpb210.conf --eNBs.[0].component_carriers.[0].drx_Config_present prRelease</Initialize_eNB_args>
</testCase>
<testCase id="030112">
<class>Initialize_eNB</class>
<desc>Initialize eNB (FDD/Band7/10MHz) with CDRX</desc>
<Initialize_eNB_args>-O ci-scripts/conf_files/enb.band7.tm1.50PRB.usrpb210.conf</Initialize_eNB_args>
</testCase>
<testCase id="030201">
<class>Terminate_eNB</class>
<desc>Terminate eNB</desc>
......@@ -64,12 +57,6 @@
<desc>Attach UE</desc>
</testCase>
<testCase id="040302">
<class>Attach_UE</class>
<desc>Attach UE</desc>
<nbMaxUEtoAttach>1</nbMaxUEtoAttach>
</testCase>
<testCase id="040401">
<class>Detach_UE</class>
<desc>Detach UE</desc>
......@@ -82,21 +69,6 @@
<ping_packetloss_threshold>5</ping_packetloss_threshold>
</testCase>
<testCase id="040512">
<class>Ping</class>
<desc>ping (10MHz - 20 sec)</desc>
<ping_args>-c 20</ping_args>
<ping_packetloss_threshold>5</ping_packetloss_threshold>
</testCase>
<testCase id="040612">
<class>Iperf</class>
<desc>iperf (10MHz - DL/30Mbps/UDP)(30 sec)(balanced profile)</desc>
<iperf_args>-u -b 30M -t 30 -i 1</iperf_args>
<iperf_packetloss_threshold>50</iperf_packetloss_threshold>
<iperf_profile>balanced</iperf_profile>
</testCase>
<testCase id="040613">
<class>Iperf</class>
<desc>iperf (10MHz - DL/30Mbps/UDP)(30 sec)(balanced profile)</desc>
......@@ -137,14 +109,6 @@
<iperf_profile>balanced</iperf_profile>
</testCase>
<testCase id="040650">
<class>Iperf</class>
<desc>iperf (10MHz - UL/20Mbps/UDP)(30 sec)(balanced profile)</desc>
<iperf_args>-u -b 20M -t 30 -i 1 -R</iperf_args>
<iperf_packetloss_threshold>50</iperf_packetloss_threshold>
<iperf_profile>balanced</iperf_profile>
</testCase>
<testCase id="040651">
<class>Iperf</class>
<desc>iperf (10MHz - UL/20Mbps/UDP)(30 sec)(balanced profile)</desc>
......
<!--
Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The OpenAirInterface Software Alliance licenses this file to You under
the OAI Public License, Version 1.1 (the "License"); you may not use this file
except in compliance with the License.
You may obtain a copy of the License at
http://www.openairinterface.org/?page_id=698
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
For more information about the OpenAirInterface (OAI) Software Alliance:
contact@openairinterface.org
-->
<testCaseList>
<htmlTabRef>test-10-cdrx-tm1</htmlTabRef>
<htmlTabName>Test-10MHz-CDRX-TM1</htmlTabName>
<htmlTabIcon>tasks</htmlTabIcon>
<repeatCount>4</repeatCount>
<TestCaseRequestedList>
030201
040101
030112 040302 040512 040612 040650 040401 040201 030201
</TestCaseRequestedList>
<TestCaseExclusionList></TestCaseExclusionList>
<testCase id="030112">
<class>Initialize_eNB</class>
<desc>Initialize eNB (FDD/Band7/10MHz) with CDRX</desc>
<Initialize_eNB_args>-O ci-scripts/conf_files/enb.band7.tm1.50PRB.usrpb210.conf --eNBs.[0].component_carriers.[0].drx_Config_present prSetup</Initialize_eNB_args>
</testCase>
<testCase id="030201">
<class>Terminate_eNB</class>
<desc>Terminate eNB</desc>
</testCase>
<testCase id="040101">
<class>Initialize_UE</class>
<desc>Initialize UE</desc>
</testCase>
<testCase id="040201">
<class>Terminate_UE</class>
<desc>Terminate UE</desc>
</testCase>
<testCase id="040302">
<class>Attach_UE</class>
<desc>Attach UE</desc>
<nbMaxUEtoAttach>1</nbMaxUEtoAttach>
</testCase>
<testCase id="040401">
<class>Detach_UE</class>
<desc>Detach UE</desc>
</testCase>
<testCase id="040512">
<class>Ping</class>
<desc>ping (10MHz - 20 sec)</desc>
<ping_args>-c 20</ping_args>
<ping_packetloss_threshold>5</ping_packetloss_threshold>
</testCase>
<testCase id="040612">
<class>Iperf</class>
<desc>iperf (10MHz - DL/30Mbps/UDP)(30 sec)(balanced profile)</desc>
<iperf_args>-u -b 30M -t 30 -i 1</iperf_args>
<iperf_packetloss_threshold>50</iperf_packetloss_threshold>
<iperf_profile>balanced</iperf_profile>
</testCase>
<testCase id="040650">
<class>Iperf</class>
<desc>iperf (10MHz - UL/20Mbps/UDP)(30 sec)(balanced profile)</desc>
<iperf_args>-u -b 20M -t 30 -i 1 -R</iperf_args>
<iperf_packetloss_threshold>50</iperf_packetloss_threshold>
<iperf_profile>balanced</iperf_profile>
</testCase>
</testCaseList>
......@@ -22,8 +22,9 @@
-->
<testCaseList>
<htmlTabRef>test-f1-05</htmlTabRef>
<htmlTabName>Test-F1-CU/DU-05MHz</htmlTabName>
<htmlTabName>Test-F1-CU-DU-05MHz</htmlTabName>
<htmlTabIcon>tasks</htmlTabIcon>
<repeatCount>4</repeatCount>
<TestCaseRequestedList>
030201
040101
......
......@@ -22,8 +22,9 @@
-->
<testCaseList>
<htmlTabRef>test-f1-10</htmlTabRef>
<htmlTabName>Test-F1-CU/DU-10MHz</htmlTabName>
<htmlTabName>Test-F1-CU-DU-10MHz</htmlTabName>
<htmlTabIcon>tasks</htmlTabIcon>
<repeatCount>4</repeatCount>
<TestCaseRequestedList>
030201
040101
......
......@@ -22,8 +22,9 @@
-->
<testCaseList>
<htmlTabRef>test-f1-20</htmlTabRef>
<htmlTabName>Test-F1-CU/DU-20MHz</htmlTabName>
<htmlTabName>Test-F1-CU-DU-20MHz</htmlTabName>
<htmlTabIcon>tasks</htmlTabIcon>
<repeatCount>4</repeatCount>
<TestCaseRequestedList>
030201
040101
......
......@@ -24,6 +24,7 @@
<htmlTabRef>test-multi-rru-10</htmlTabRef>
<htmlTabName>Test-Multi-RRU-10MHz</htmlTabName>
<htmlTabIcon>tasks</htmlTabIcon>
<repeatCount>4</repeatCount>
<TestCaseRequestedList>
030211 030212 030213
030111 030112 030113
......
......@@ -24,6 +24,7 @@
<htmlTabRef>test-10mhz-orange</htmlTabRef>
<htmlTabName>Test-10Mhz-Orange</htmlTabName>
<htmlTabIcon>tasks</htmlTabIcon>
<repeatCount>4</repeatCount>
<TestCaseRequestedList>
090109
090110 000001 090109
......
......@@ -24,6 +24,7 @@
<htmlTabRef>test-10mHz-sfr</htmlTabRef>
<htmlTabName>Test-10MHz-SFR</htmlTabName>
<htmlTabIcon>tasks</htmlTabIcon>
<repeatCount>4</repeatCount>
<TestCaseRequestedList>
090109
090111 000001 090109
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment