Skip to content

O
OpenXG UE
Commit 163e0050 authored by Mohammed Ismail's avatar Mohammed Ismail
Browse files
Options

CI: fix on phySim 

Signed-off-by: default avatarMohammed Ismail <mohammed.ismail@openairinterface.org>
parent c93bc244
Hide whitespace changes
Inline Side-by-side
Showing with 140 additions and 41 deletions
ci-scripts/Jenkinsfile-physim-deploy
View file @ 163e0050
...@@ -207,5 +207,24 @@ pipeline { ...@@ -207,5 +207,24 @@ pipeline {
} }
} }
} }
} stage ("Log Collection") {
steps {
script {
dir ('ci-scripts') {
// Zipping all archived log files
sh "zip -r -qq physim_deploytest_logs_${env.BUILD_ID}.zip physim_test_logs_*/*"
sh "rm -rf physim_test_logs_*/"
if (fileExists("physim_deploytest_logs_${env.BUILD_ID}.zip")) {
archiveArtifacts artifacts: "physim_deploytest_logs_${env.BUILD_ID}.zip"
}
if (fileExists("test_results.html")) {
sh "mv test_results.html test_results_${env.JOB_NAME}.html"
sh "sed -i -e 's#TEMPLATE_JOB_NAME#${JOB_NAME}#' -e 's@build #TEMPLATE_BUILD_ID@build #${BUILD_ID}@' -e 's#Build-ID: TEMPLATE_BUILD_ID#Build-ID: <a href=\"${BUILD_URL}\">${BUILD_ID}</a>#' -e 's#TEMPLATE_STAGE_NAME#${testStageName}#' test_results_${JOB_NAME}.html"
archiveArtifacts artifacts: "test_results_${env.JOB_NAME}.html"
}
}
}
}
}
}
} }
\ No newline at end of file
ci-scripts/cls_physim1.py
View file @ 163e0050
...@@ -55,6 +55,9 @@ class PhySim: ...@@ -55,6 +55,9 @@ class PhySim:
self.ranCommitID= "" self.ranCommitID= ""
self.ranAllowMerge= "" self.ranAllowMerge= ""
self.ranTargetBranch= "" self.ranTargetBranch= ""
self.testResult = {}
self.testCount = [0,0,0]
self.testSummary = {}
self.exitStatus=0 self.exitStatus=0
#-----------------$ #-----------------$
...@@ -101,7 +104,8 @@ class PhySim: ...@@ -101,7 +104,8 @@ class PhySim:
if self.ranCommitID != '': if self.ranCommitID != '':
mySSH.command('git checkout -f ' + self.ranCommitID, '\$', 5) mySSH.command('git checkout -f ' + self.ranCommitID, '\$', 5)
mySSH.command("sudo podman image inspect --format='Size = {{.Size}} bytes' oai-physim:ci-temp", '\$', 60) # Check if image is exist on the Red Hat server, before pushing it to OC cluster
mySSH.command("sudo podman image inspect --format='Size = {{.Size}} bytes' oai-physim:temp", '\$', 60)
if mySSH.getBefore().count('no such image') != 0: if mySSH.getBefore().count('no such image') != 0:
logging.error('\u001B[1m No such image oai-physim\u001B[0m') logging.error('\u001B[1m No such image oai-physim\u001B[0m')
sys.exit(-1) sys.exit(-1)
...@@ -122,9 +126,8 @@ class PhySim: ...@@ -122,9 +126,8 @@ class PhySim:
else: else:
logging.debug('oai-physim size is unknown') logging.debug('oai-physim size is unknown')
# logging to OC cluster # logging to OC Cluster and then switch to corresponding project
mySSH.command(f'oc login -u {ocUserName} -p {ocPassword}', '\$', 6) mySSH.command(f'oc login -u {ocUserName} -p {ocPassword}', '\$', 6)
print(mySSH.getBefore())
if mySSH.getBefore().count('Login successful.') == 0: if mySSH.getBefore().count('Login successful.') == 0:
logging.error('\u001B[1m OC Cluster Login Failed\u001B[0m') logging.error('\u001B[1m OC Cluster Login Failed\u001B[0m')
sys.exit(-1) sys.exit(-1)
...@@ -136,25 +139,51 @@ class PhySim: ...@@ -136,25 +139,51 @@ class PhySim:
sys.exit(-1) sys.exit(-1)
else: else:
logging.debug(f'\u001B[1m Now using project {ocWorkspace}\u001B[0m') logging.debug(f'\u001B[1m Now using project {ocWorkspace}\u001B[0m')
# Tag the image and push to the OC cluster
mySSH.command('oc whoami -t | sudo podman login -u ' + ocUserName + ' --password-stdin https://default-route-openshift-image-registry.apps.5glab.nsa.eurecom.fr/ --tls-verify=false', '\$', 6)
if mySSH.getBefore().count('Login Succeeded!') == 0:
logging.error('\u001B[1m Podman Login to OC Cluster Registry Failed\u001B[0m')
sys.exit(-1)
else:
logging.debug('\u001B[1m Podman Login to OC Cluster Registry Successfully\u001B[0m')
mySSH.command(f'sudo podman tag oai-physim:temp default-route-openshift-image-registry.apps.5glab.nsa.eurecom.fr/{self.OCWorkspace}/oai-physim:temp', '\$', 6)
mySSH.command(f'sudo podman push default-route-openshift-image-registry.apps.5glab.nsa.eurecom.fr/{self.OCWorkspace}/oai-physim:temp --tls-verify=false', '\$', 6)
if mySSH.getBefore().count('Storing signatures') == 0:
logging.error('\u001B[1m Image "oai-physim" push to OC Cluster Registry Failed\u001B[0m')
sys.exit(-1)
else:
logging.debug('\u001B[1m Image "oai-physim" push to OC Cluster Registry Successfully\u001B[0m')
# Using helm charts deployment # Using helm charts deployment
mySSH.command('helm install physim ./charts/physims/', '\$', 6) mySSH.command('helm install physim ./charts/physims/ | tee -a cmake_targets/log/physim_helm_summary.txt 2>&1', '\$', 6)
if mySSH.getBefore().count('STATUS: deployed') == 0: if mySSH.getBefore().count('STATUS: deployed') == 0:
logging.error('\u001B[1m Deploying PhySim Failed using helm chart\u001B[0m') logging.error('\u001B[1m Deploying PhySim Failed using helm chart on OC Cluster\u001B[0m')
mySSH.command('helm uninstall physim >> cmake_targets/log/physim_helm_summary.txt 2>&1', '\$', 6)
self.AnalyzeLogFile_phySim(HTML)
sys.exit(-1) sys.exit(-1)
else: else:
logging.debug('\u001B[1m Deployed PhySim Successfully using helm chart\u001B[0m') logging.debug('\u001B[1m Deployed PhySim Successfully using helm chart\u001B[0m')
isRunning = False isRunning = False
while(isRunning == False): count = 0
while(count < 2 and isRunning == False):
time.sleep(60)
mySSH.command('oc get pods -l app.kubernetes.io/instance=physim', '\$', 6) mySSH.command('oc get pods -l app.kubernetes.io/instance=physim', '\$', 6)
if mySSH.getBefore().count('Running') == 12: if mySSH.getBefore().count('Running') == 12:
logging.debug('\u001B[1m Running the physim test Scenarios\u001B[0m') logging.debug('\u001B[1m Running the physim test Scenarios\u001B[0m')
isRunning = True isRunning = True
podNames = re.findall('oai[\S\d\w]+', mySSH.getBefore()) podNames = re.findall('oai[\S\d\w]+', mySSH.getBefore())
count +=1
if isRunning == False:
logging.error('\u001B[1m Some PODS Running FAILED \u001B[0m')
mySSH.command('oc get pods -l app.kubernetes.io/instance=physim > cmake_targets/log/physim_pods_summary.txt 2>&1', '\$', 6)
mySSH.command('helm uninstall physim >> cmake_targets/log/physim_helm_summary.txt 2>&1', '\$', 6)
self.AnalyzeLogFile_phySim(HTML)
sys.exit(-1)
# Waiting to complete the running test # Waiting to complete the running test
count = 0 count = 0
isFinished = False isFinished = False
while(count < 25 or isFinished == False): while(count < 25 and isFinished == False):
time.sleep(60) time.sleep(60)
mySSH.command('oc get pods -l app.kubernetes.io/instance=physim', '\$', 6) mySSH.command('oc get pods -l app.kubernetes.io/instance=physim', '\$', 6)
result = re.search('oai-nr-dlsim[\S\d\w]+', mySSH.getBefore()) result = re.search('oai-nr-dlsim[\S\d\w]+', mySSH.getBefore())
...@@ -170,38 +199,61 @@ class PhySim: ...@@ -170,38 +199,61 @@ class PhySim:
# Getting the logs of each executables running in individual pods # Getting the logs of each executables running in individual pods
for podName in podNames: for podName in podNames:
mySSH.command(f'oc logs {podName} >> cmake_targets/log/physim_test.txt 2>&1', '\$', 6) mySSH.command(f'oc logs {podName} >> cmake_targets/log/physim_test.txt 2>&1', '\$', 6)
mySSH.command('cd ' + lSourcePath + '/cmake_targets', '\$', 5)
mySSH.command('mkdir -p physim_test_log_' + self.testCase_id, '\$', 5)
mySSH.command('mv log/* ' + 'physim_test_log_' + self.testCase_id, '\$', 5)
# UnDeploy the physical simulator pods # UnDeploy the physical simulator pods
mySSH.command('helm uninstall physim', '\$', 10) mySSH.command('helm uninstall physim | tee -a cmake_targets/log/physim_helm_summary.txt 2>&1', '\$', 6)
if mySSH.getBefore().count('release "physim" uninstalled') != 0: if mySSH.getBefore().count('release "physim" uninstalled') != 0:
logging.debug('\u001B[1m UnDeployed PhySim Successfully on OC Cluster\u001B[0m') logging.debug('\u001B[1m UnDeployed PhySim Successfully on OC Cluster\u001B[0m')
else: else:
logging.debug('\u001B[1m Failed to UnDeploy PhySim on OC Cluster\u001B[0m') logging.debug('\u001B[1m Failed to UnDeploy PhySim on OC Cluster\u001B[0m')
mySSH.command('oc logout', '\$', 6)
mySSH.close()
self.AnalyzeLogFile_phySim(HTML)
def AnalyzeLogFile_phySim(self, HTML):
lIpAddr = self.eNBIPAddress
lUserName = self.eNBUserName
lPassWord = self.eNBPassword
lSourcePath = self.eNBSourceCodePath
mySSH = SSH.SSHConnection()
mySSH.open(lIpAddr, lUserName, lPassWord)
mySSH.command('cd ' + lSourcePath, '\$', 5)
mySSH.command('cd ' + lSourcePath + '/cmake_targets', '\$', 5)
mySSH.command('mkdir -p physim_test_log_' + self.testCase_id, '\$', 5)
mySSH.command('mv log/physim_* ' + 'physim_test_log_' + self.testCase_id, '\$', 5)
if not os.path.exists(f'./physim_test_logs_{self.testCase_id}'):
os.mkdir(f'./physim_test_logs_{self.testCase_id}')
mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/physim_test_log_' + self.testCase_id + '/*', './physim_test_logs_' + self.testCase_id)
mySSH.command('rm -rf ./physim_test_log_'+ self.testCase_id, '\$', 5)
mySSH.command('oc logout', '\$', 6)
mySSH.close() mySSH.close()
mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/physim_test_log_' + self.testCase_id + '/*', '.') # physim test log analysis
testResult = {} nextt = 0
testCount = 0 if (os.path.isfile(f'./physim_test_logs_{self.testCase_id}/physim_test.txt')):
if (os.path.isfile('./physim_test.txt')): with open(f'./physim_test_logs_{self.testCase_id}/physim_test.txt', 'r') as logfile:
with open('./physim_test.txt', 'r') as logfile:
for line in logfile: for line in logfile:
ret = re.search('execution', str(line)) if re.search('execution 015', str(line)) or re.search('Bypassing compilation', str(line)):
if ret is not None: nextt = 1
testCount += 1 elif nextt == 1:
if not re.search('Test Results', str(line)):
nextt = 0
ret2 = re.search('T[^\n]*', str(line))
if ret2 is not None:
ret3 = ret2.group()
if re.search('execution 015', str(line)):
self.testCount[0] += 1
testName = line.split() testName = line.split()
ret1 = re.search('Result = PASS', str(line)) ret1 = re.search('Result = PASS', str(line))
if ret1 is not None: if ret1 is not None:
testResult[testName[1]] = 'PASS' self.testResult[testName[1]] = [ret3, 'PASS']
self.testCount[1] += 1
else: else:
testResult[testName[1]] = 'FAIL' self.testResult[testName[1]] = [ret3, 'FAIL']
self.testCount[2] += 1
self.testSummary['Nbtests'] = self.testCount[0]
self.testSummary['Nbpass'] = self.testCount[1]
self.testSummary['Nbfail'] = self.testCount[2]
HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK) HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK)
HTML.CreateHtmlTestRowPhySimTestResult(testResult) HTML.CreateHtmlTestRowPhySimTestResult(self.testSummary,self.testResult)
logging.info('\u001B[1m Physical Simulator Pass\u001B[0m') logging.info('\u001B[1m Physical Simulator Pass\u001B[0m')
return 0
return 0 \ No newline at end of file
ci-scripts/html.py
View file @ 163e0050
...@@ -563,21 +563,49 @@ class HTMLManagement(): ...@@ -563,21 +563,49 @@ class HTMLManagement():
self.htmlFile.write(' </tr>\n') self.htmlFile.write(' </tr>\n')
vId += 1 vId += 1
def CreateHtmlTestRowPhySimTestResult(self, testResult): def CreateHtmlTestRowPhySimTestResult(self, testSummary, testResult):
if (self.htmlFooterCreated or (not self.htmlHeaderCreated)): if (self.htmlFooterCreated or (not self.htmlHeaderCreated)):
return return
self.htmlFile = open('test_results.html', 'a') self.htmlFile = open('test_results.html', 'a')
if bool(testResult) == False and bool(testSummary) == False:
self.htmlFile.write(' <tr bgcolor = "red" >\n')
self.htmlFile.write(' <td colspan=' + str(5+self.htmlUEConnected) + '><b> ----PHYSIM TESTING FAILED - Unable to recover the test logs ---- </b></td>\n')
self.htmlFile.write(' </tr>\n')
else:
# Tab header # Tab header
self.htmlFile.write(' <tr bgcolor = "#33CCFF" >\n') self.htmlFile.write(' <tr bgcolor = "#F0F0F0" >\n')
self.htmlFile.write(' <th colspan="2">Test Description</th>\n') self.htmlFile.write(' <td colspan=' + str(5+self.htmlUEConnected) + '><b> ---- PHYSIM TEST SUMMARY---- </b></td>\n')
self.htmlFile.write(' <th>Result</th>\n') self.htmlFile.write(' </tr>\n')
self.htmlFile.write(' </tr>\n') self.htmlFile.write(' <tr bgcolor = "#33CCFF" >\n')
for key, value in testResult.items(): self.htmlFile.write(' <th colspan="2">LogFile Name</th>\n')
self.htmlFile.write(' <th colspan="2">Nb Tests</th>\n')
self.htmlFile.write(' <th>Nb Failure</th>\n')
self.htmlFile.write(' <th>Nb Pass</th>\n')
self.htmlFile.write(' </tr>\n')
self.htmlFile.write(' <tr>\n') self.htmlFile.write(' <tr>\n')
self.htmlFile.write(' <td colspan="2" bgcolor = "lightcyan" >' + key + ' </td>\n') self.htmlFile.write(' <td colspan="2" bgcolor = "lightcyan" > physim_test.txt </td>\n')
if value == 'PASS': self.htmlFile.write(' <td colspan="2" bgcolor = "lightcyan" >' + str(testSummary['Nbtests']) + ' </td>\n')
self.htmlFile.write(' <td bgcolor = "green" >' + value + '</td>\n') if testSummary['Nbfail'] == 0:
self.htmlFile.write(' <td bgcolor = "lightcyan" >' + str(testSummary['Nbfail']) + ' </td>\n')
else: else:
self.htmlFile.write(' <td bgcolor = "red" >' + value + '</td>\n') self.htmlFile.write(' <td bgcolor = "red" >' + str(testSummary['Nbfail']) + ' </td>\n')
self.htmlFile.write(' <td gcolor = "lightcyan" >' + str(testSummary['Nbpass']) + ' </td>\n')
self.htmlFile.write(' </tr>\n')
self.htmlFile.write(' <tr bgcolor = "#F0F0F0" >\n')
self.htmlFile.write(' <td colspan=' + str(5+self.htmlUEConnected) + '><b> ---- PHYSIM TEST DETAIL INFO---- </b></td>\n')
self.htmlFile.write(' </tr>\n')
self.htmlFile.write(' <tr bgcolor = "#33CCFF" >\n')
self.htmlFile.write(' <th colspan="2">Test Name</th>\n')
self.htmlFile.write(' <th colspan="2">Test Description</th>\n')
self.htmlFile.write(' <th colspan=' + str(1+self.htmlUEConnected) + '>Result</th>\n')
self.htmlFile.write(' </tr>\n')
for key, value in testResult.items():
self.htmlFile.write(' <tr>\n')
self.htmlFile.write(' <td colspan="2" bgcolor = "lightcyan" >' + key + ' </td>\n')
self.htmlFile.write(' <td colspan="2" bgcolor = "lightcyan" >' + value[0] + '</td>\n')
if 'PASS' in value:
self.htmlFile.write(' <td colspan=' + str(1+self.htmlUEConnected) + ' bgcolor = "green" >' + value[1] + '</td>\n')
else:
self.htmlFile.write(' <td colspan=' + str(1+self.htmlUEConnected) + ' bgcolor = "red" >' + value[1] + '</td>\n')
self.htmlFile.close() self.htmlFile.close()
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment