Commit 43a2ae00 authored by Raphael Defosseux's avatar Raphael Defosseux

style(ci): using and debugging the new LocalCmd class

Signed-off-by: default avatarRaphael Defosseux <raphael.defosseux@eurecom.fr>
parent 6905159a
...@@ -74,32 +74,27 @@ class Cmd(metaclass=abc.ABCMeta): ...@@ -74,32 +74,27 @@ class Cmd(metaclass=abc.ABCMeta):
class LocalCmd(Cmd): class LocalCmd(Cmd):
def __init__(self, d = None): def __init__(self, d = None):
self.cwd = d self.cwd = d
if self.cwd is not None:
logging.debug(f'Working dir will be {self.cwd}')
self.cp = sp.CompletedProcess(args='', returncode=0, stdout='') self.cp = sp.CompletedProcess(args='', returncode=0, stdout='')
def run(self, line, timeout=300, silent=False, reportNonZero=True): def run(self, line, timeout=300, silent=False, reportNonZero=True):
if type(line) is str:
line = [s for s in line.split(' ') if len(s) > 0]
if not silent: if not silent:
logging.debug(' '.join(line)) logging.info(line)
try: try:
ret = sp.run(line, cwd=self.cwd, stdout=sp.PIPE, stderr=sp.STDOUT, timeout=timeout) ret = sp.run(line, shell=True, cwd=self.cwd, stdout=sp.PIPE, stderr=sp.STDOUT, timeout=timeout)
except Exception as e: except Exception as e:
ret = sp.CompletedProcess(args=line, returncode=255, stdout=f'Exception: {str(e)}'.encode('utf-8')) ret = sp.CompletedProcess(args=line, returncode=255, stdout=f'Exception: {str(e)}'.encode('utf-8'))
if ret.stdout is None: if ret.stdout is None:
ret.stdout = b'' ret.stdout = b''
ret.stdout = ret.stdout.decode('utf-8').strip() ret.stdout = ret.stdout.decode('utf-8').strip()
if reportNonZero and ret.returncode != 0: if reportNonZero and ret.returncode != 0:
cmd = ' '.join(ret.args) logging.warning(f'command "{ret.args}" returned non-zero returncode {ret.returncode}: output:\n{ret.stdout}')
logging.warning(f'command "{cmd}" returned non-zero returncode {ret.returncode}: output:\n{ret.stdout}')
self.cp = ret self.cp = ret
return ret return ret
def command(self, commandline, expectedline=None, timeout=300, silent=False, resync=False): def command(self, commandline, expectedline=None, timeout=300, silent=False, resync=False):
line = [s for s in commandline.split(' ') if len(s) > 0] self.run(commandline, timeout, silent)
if line[0] == 'cd':
self.cd(line[1], silent)
else:
self.run(line, timeout, silent)
return 0 return 0
def close(self): def close(self):
......
...@@ -40,6 +40,7 @@ import subprocess ...@@ -40,6 +40,7 @@ import subprocess
import time import time
import pyshark import pyshark
import threading import threading
import cls_cmd
from multiprocessing import Process, Lock, SimpleQueue from multiprocessing import Process, Lock, SimpleQueue
from zipfile import ZipFile from zipfile import ZipFile
...@@ -732,57 +733,42 @@ class Containerize(): ...@@ -732,57 +733,42 @@ class Containerize():
sys.exit('Insufficient Parameter') sys.exit('Insufficient Parameter')
if lIpAddr != 'none': if lIpAddr != 'none':
logging.debug('Pulling images onto server: ' + lIpAddr) logging.debug('Pulling images onto server: ' + lIpAddr)
mySSH = SSH.SSHConnection() myCmd = cls_cmd.RemoteCmd(lIpAddr)
mySSH.open(lIpAddr, lUserName, lPassWord)
else: else:
logging.debug('Pulling images locally') logging.debug('Pulling images locally')
myCmd = cls_cmd.LocalCmd()
cmd = 'docker login -u oaicicd -p oaicicd porcepix.sboai.cs.eurecom.fr' cmd = 'docker login -u oaicicd -p oaicicd porcepix.sboai.cs.eurecom.fr'
if lIpAddr != 'none': response = myCmd.run(cmd)
mySSH.command(cmd, '\$', 5) if re.search('Login Succeeded', response.stdout) is None:
response = mySSH.getBefore()
else:
response = self.run_cmd_local_host(cmd)
if re.search('Login Succeeded', response) is None:
msg = 'Could not log into local registry' msg = 'Could not log into local registry'
logging.error(msg) logging.error(msg)
if lIpAddr != 'none': myCmd.close()
mySSH.close()
HTML.CreateHtmlTestRow(msg, 'KO', CONST.ALL_PROCESSES_OK) HTML.CreateHtmlTestRow(msg, 'KO', CONST.ALL_PROCESSES_OK)
return False return False
for image in self.imageToPull: for image in self.imageToPull:
tagToUse = self.ImageTagToUse(image) tagToUse = self.ImageTagToUse(image)
cmd = f'docker pull {tagToUse}' cmd = f'docker pull {tagToUse}'
if lIpAddr != 'none': response = myCmd.run(cmd, timeout=120)
mySSH.command(cmd, '\$', 120) if re.search('Status: Downloaded newer image for |Status: Image is up to date for', response.stdout) is None:
response = mySSH.getBefore()
else:
response = self.run_cmd_local_host(cmd)
if re.search('Status: Downloaded newer image for |Status: Image is up to date for', response) is None:
logging.debug(response) logging.debug(response)
msg = f'Could not pull {image} from local registry : {tagToUse}' msg = f'Could not pull {image} from local registry : {tagToUse}'
logging.error(msg) logging.error(msg)
if lIpAddr != 'none': myCmd.close()
mySSH.close()
HTML.CreateHtmlTestRow('msg', 'KO', CONST.ALL_PROCESSES_OK) HTML.CreateHtmlTestRow('msg', 'KO', CONST.ALL_PROCESSES_OK)
return False return False
cmd = 'docker logout porcepix.sboai.cs.eurecom.fr' cmd = 'docker logout porcepix.sboai.cs.eurecom.fr'
if lIpAddr != 'none': response = myCmd.run(cmd)
mySSH.command(cmd, '\$', 5) if re.search('Removing login credentials', response.stdout) is None:
response = mySSH.getBefore()
else:
response = self.run_cmd_local_host(cmd)
if re.search('Removing login credentials', response) is None:
msg = 'Could not log off from local registry' msg = 'Could not log off from local registry'
logging.error(msg) logging.error(msg)
mySSH.close() myCmd.close()
HTML.CreateHtmlTestRow(msg, 'KO', CONST.ALL_PROCESSES_OK) HTML.CreateHtmlTestRow(msg, 'KO', CONST.ALL_PROCESSES_OK)
return False return False
if lIpAddr != 'none': myCmd.close()
mySSH.close()
HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK) HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK)
return True return True
...@@ -809,21 +795,17 @@ class Containerize(): ...@@ -809,21 +795,17 @@ class Containerize():
sys.exit('Insufficient Parameter') sys.exit('Insufficient Parameter')
if lIpAddr != 'none': if lIpAddr != 'none':
logging.debug('Removing test images from server: ' + lIpAddr) logging.debug('Removing test images from server: ' + lIpAddr)
mySSH = SSH.SSHConnection() myCmd = cls_cmd.RemoteCmd(lIpAddr)
mySSH.open(lIpAddr, lUserName, lPassWord)
else: else:
logging.debug('Removing test images locally') logging.debug('Removing test images locally')
myCmd = cls_cmd.LocalCmd()
imageNames = ['oai-enb', 'oai-gnb', 'oai-lte-ue', 'oai-nr-ue', 'oai-lte-ru'] imageNames = ['oai-enb', 'oai-gnb', 'oai-lte-ue', 'oai-nr-ue', 'oai-lte-ru']
for image in imageNames: for image in imageNames:
cmd = f'docker rmi {self.ImageTagToUse(image)} || true' cmd = f'docker rmi {self.ImageTagToUse(image)}'
if lIpAddr != 'none': myCmd.run(cmd, reportNonZero=False)
mySSH.command(cmd, '\$', 5)
else:
self.run_cmd_local_host(cmd)
if lIpAddr != 'none': myCmd.close()
mySSH.close()
HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK) HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK)
return True return True
...@@ -1046,42 +1028,45 @@ class Containerize(): ...@@ -1046,42 +1028,45 @@ class Containerize():
def DeployGenObject(self, HTML, RAN, UE): def DeployGenObject(self, HTML, RAN, UE):
self.exitStatus = 0 self.exitStatus = 0
logging.debug('\u001B[1m Checking Services to deploy\u001B[0m') logging.debug('\u001B[1m Checking Services to deploy\u001B[0m')
cmd = 'cd ' + self.yamlPath[0] + ' && docker-compose config --services' # Implicitly we are running locally
try: myCmd = cls_cmd.LocalCmd(d = self.yamlPath[0])
listServices = self.run_cmd_local_host(cmd) cmd = 'docker-compose config --services'
except Exception as e: listServices = myCmd.run(cmd)
if listServices.returncode != 0:
myCmd.close()
self.exitStatus = 1 self.exitStatus = 1
HTML.CreateHtmlTestRow('SVC not Found', 'KO', CONST.ALL_PROCESSES_OK) HTML.CreateHtmlTestRow('SVC not Found', 'KO', CONST.ALL_PROCESSES_OK)
return return
for reqSvc in self.services[0].split(' '): for reqSvc in self.services[0].split(' '):
res = re.search(reqSvc, listServices) res = re.search(reqSvc, listServices.stdout)
if res is None: if res is None:
logging.error(reqSvc + ' not found in specified docker-compose') logging.error(reqSvc + ' not found in specified docker-compose')
self.exitStatus = 1 self.exitStatus = 1
if (self.exitStatus == 1): if (self.exitStatus == 1):
myCmd.close()
HTML.CreateHtmlTestRow('SVC not Found', 'KO', CONST.ALL_PROCESSES_OK) HTML.CreateHtmlTestRow('SVC not Found', 'KO', CONST.ALL_PROCESSES_OK)
return return
cmd = 'cd ' + self.yamlPath[0] + ' && cp docker-compose.y*ml docker-compose-ci.yml' cmd = 'cp docker-compose.y*ml docker-compose-ci.yml'
self.run_cmd_local_host(cmd, silent=self.displayedNewTags) myCmd.command(cmd, silent=self.displayedNewTags)
imageNames = ['oai-enb', 'oai-gnb', 'oai-lte-ue', 'oai-nr-ue', 'oai-lte-ru'] imageNames = ['oai-enb', 'oai-gnb', 'oai-lte-ue', 'oai-nr-ue', 'oai-lte-ru']
for image in imageNames: for image in imageNames:
tagToUse = self.ImageTagToUse(image) tagToUse = self.ImageTagToUse(image)
cmd = f'cd {self.yamlPath[0]} && sed -i -e "s@oaisoftwarealliance/{image}:develop@{tagToUse}@" docker-compose-ci.yml' cmd = f'sed -i -e "s@oaisoftwarealliance/{image}:develop@{tagToUse}@" docker-compose-ci.yml'
self.run_cmd_local_host(cmd, silent=self.displayedNewTags) myCmd.command(cmd, silent=self.displayedNewTags)
silent=self.displayedNewTags = True self.displayedNewTags = True
cmd = 'cd ' + self.yamlPath[0] + ' && docker-compose -f docker-compose-ci.yml up -d ' + self.services[0] cmd = 'docker-compose -f docker-compose-ci.yml up -d ' + self.services[0]
try: deployStatus = myCmd.run(cmd, timeout=100)
deployStatus = self.run_cmd_local_host(cmd, 100) if deployStatus.returncode != 0:
except Exception as e: myCmd.close()
self.exitStatus = 1 self.exitStatus = 1
logging.error('Could not deploy') logging.error('Could not deploy')
HTML.CreateHtmlTestRow('Could not deploy', 'KO', CONST.ALL_PROCESSES_OK) HTML.CreateHtmlTestRow('Could not deploy', 'KO', CONST.ALL_PROCESSES_OK)
return return
logging.debug('\u001B[1m Checking if all deployed healthy\u001B[0m') logging.debug('\u001B[1m Checking if all deployed healthy\u001B[0m')
cmd = 'cd ' + self.yamlPath[0] + ' && docker-compose -f docker-compose-ci.yml ps -a' cmd = 'docker-compose -f docker-compose-ci.yml ps -a'
count = 0 count = 0
healthy = 0 healthy = 0
restarting = 0 restarting = 0
...@@ -1089,10 +1074,10 @@ class Containerize(): ...@@ -1089,10 +1074,10 @@ class Containerize():
while (count < 10): while (count < 10):
count += 1 count += 1
containerStatus = [] containerStatus = []
deployStatus = self.run_cmd_local_host(cmd, 30, silent=True) deployStatus = myCmd.run(cmd, 30, silent=True)
healthy = 0 healthy = 0
restarting = 0 restarting = 0
for state in deployStatus.split('\n'): for state in deployStatus.stdout.split('\n'):
state = state.strip() state = state.strip()
res = re.search('Name|NAME|----------', state) res = re.search('Name|NAME|----------', state)
if res is not None: if res is not None:
...@@ -1112,7 +1097,7 @@ class Containerize(): ...@@ -1112,7 +1097,7 @@ class Containerize():
if re.search('\(healthy\)', state) is not None: if re.search('\(healthy\)', state) is not None:
healthy += 1 healthy += 1
if re.search('rfsim4g-db-init.*Exit 0', state) is not None or re.search('rfsim4g-db-init.*Exited \(0\)', state) is not None: if re.search('rfsim4g-db-init.*Exit 0', state) is not None or re.search('rfsim4g-db-init.*Exited \(0\)', state) is not None:
self.run_cmd_local_host('docker rm -f rfsim4g-db-init || true', 30, silent=True) myCmd.run('docker rm -f rfsim4g-db-init', timeout=30, silent=True, reportNonZero=False)
if re.search('Restarting', state) is None: if re.search('Restarting', state) is None:
containerStatus.append(state) containerStatus.append(state)
else: else:
...@@ -1127,10 +1112,12 @@ class Containerize(): ...@@ -1127,10 +1112,12 @@ class Containerize():
if newCont == 'rfsim4g-db-init': if newCont == 'rfsim4g-db-init':
continue continue
cmd = 'docker inspect -f "{{.Config.Image}}" ' + newCont cmd = 'docker inspect -f "{{.Config.Image}}" ' + newCont
imageName = self.run_cmd_local_host(cmd, 30, silent=True) imageInspect = myCmd.run(cmd, timeout=30, silent=True)
imageName = str(imageName).strip() imageName = str(imageInspect.stdout).strip()
cmd = 'docker image inspect --format "{{.RepoTags}}\t{{.Size}} bytes\t{{.Created}}\t{{.Id}}" ' + imageName cmd = 'docker image inspect --format "{{.RepoTags}}\t{{.Size}} bytes\t{{.Created}}\t{{.Id}}" ' + imageName
imagesInfo += self.run_cmd_local_host(cmd, 30, silent=True) imageInspect = myCmd.run(cmd, 30, silent=True)
imagesInfo += imageInspect.stdout.strip()
myCmd.close()
html_queue = SimpleQueue() html_queue = SimpleQueue()
html_cell = '<pre style="background-color:white">\n' html_cell = '<pre style="background-color:white">\n'
...@@ -1168,8 +1155,10 @@ class Containerize(): ...@@ -1168,8 +1155,10 @@ class Containerize():
pass pass
def CaptureOnDockerNetworks(self): def CaptureOnDockerNetworks(self):
cmd = 'cd ' + self.yamlPath[0] + ' && docker-compose -f docker-compose-ci.yml config | grep com.docker.network.bridge.name | sed -e "s@^.*name: @@"' myCmd = cls_cmd.LocalCmd(d = self.yamlPath[0])
networkNames = self.run_cmd_local_host(cmd, silent=True) cmd = 'docker-compose -f docker-compose-ci.yml config | grep com.docker.network.bridge.name | sed -e "s@^.*name: @@"'
networkNames = myCmd.run(cmd, silent=True)
myCmd.close()
if re.search('4g.*rfsimulator', self.yamlPath[0]) is not None: if re.search('4g.*rfsimulator', self.yamlPath[0]) is not None:
# Excluding any traffic from LTE-UE container (192.168.61.30) # Excluding any traffic from LTE-UE container (192.168.61.30)
# From the trf-gen, keeping only PING traffic # From the trf-gen, keeping only PING traffic
...@@ -1183,7 +1172,7 @@ class Containerize(): ...@@ -1183,7 +1172,7 @@ class Containerize():
else: else:
return return
interfaces = [] interfaces = []
for name in networkNames.split('\n'): for name in networkNames.stdout.split('\n'):
if re.search('rfsim', name) is not None or re.search('l2sim', name) is not None: if re.search('rfsim', name) is not None or re.search('l2sim', name) is not None:
interfaces.append(name) interfaces.append(name)
ymlPath = self.yamlPath[0].split('/') ymlPath = self.yamlPath[0].split('/')
...@@ -1195,28 +1184,30 @@ class Containerize(): ...@@ -1195,28 +1184,30 @@ class Containerize():
def UndeployGenObject(self, HTML, RAN, UE): def UndeployGenObject(self, HTML, RAN, UE):
self.exitStatus = 0 self.exitStatus = 0
# Implicitly we are running locally
ymlPath = self.yamlPath[0].split('/') ymlPath = self.yamlPath[0].split('/')
logPath = '../cmake_targets/log/' + ymlPath[1] logPath = '../cmake_targets/log/' + ymlPath[1]
myCmd = cls_cmd.LocalCmd(d = self.yamlPath[0])
cmd = 'cd ' + self.yamlPath[0] + ' && cp docker-compose.y*ml docker-compose-ci.yml' cmd = 'cp docker-compose.y*ml docker-compose-ci.yml'
self.run_cmd_local_host(cmd, silent=self.displayedNewTags) myCmd.command(cmd, silent=self.displayedNewTags)
imageNames = ['oai-enb', 'oai-gnb', 'oai-lte-ue', 'oai-nr-ue', 'oai-lte-ru'] imageNames = ['oai-enb', 'oai-gnb', 'oai-lte-ue', 'oai-nr-ue', 'oai-lte-ru']
for image in imageNames: for image in imageNames:
tagToUse = self.ImageTagToUse(image) tagToUse = self.ImageTagToUse(image)
cmd = f'cd {self.yamlPath[0]} && sed -i -e "s@oaisoftwarealliance/{image}:develop@{tagToUse}@" docker-compose-ci.yml' cmd = f'sed -i -e "s@oaisoftwarealliance/{image}:develop@{tagToUse}@" docker-compose-ci.yml'
self.run_cmd_local_host(cmd, silent=self.displayedNewTags) myCmd.command(cmd, silent=self.displayedNewTags)
silent=self.displayedNewTags = True self.displayedNewTags = True
# check which containers are running for log recovery later # check which containers are running for log recovery later
cmd = 'cd ' + self.yamlPath[0] + ' && docker-compose -f docker-compose-ci.yml ps --all' cmd = 'docker-compose -f docker-compose-ci.yml ps --all'
deployStatusLogs = self.run_cmd_local_host(cmd, 30) deployStatusLogs = myCmd.run(cmd, timeout=30)
# Stop the containers to shut down objects # Stop the containers to shut down objects
logging.debug('\u001B[1m Stopping containers \u001B[0m') logging.debug('\u001B[1m Stopping containers \u001B[0m')
cmd = 'cd ' + self.yamlPath[0] + ' && docker-compose -f docker-compose-ci.yml stop -t3' cmd = 'docker-compose -f docker-compose-ci.yml stop -t3'
try: deployStatus = myCmd.run(cmd, timeout=100)
deployStatus = self.run_cmd_local_host(cmd, 100) if deployStatus.returncode != 0:
except Exception as e: myCmd.close()
self.exitStatus = 1 self.exitStatus = 1
logging.error('Could not stop containers') logging.error('Could not stop containers')
HTML.CreateHtmlTestRow('Could not stop', 'KO', CONST.ALL_PROCESSES_OK) HTML.CreateHtmlTestRow('Could not stop', 'KO', CONST.ALL_PROCESSES_OK)
...@@ -1224,7 +1215,12 @@ class Containerize(): ...@@ -1224,7 +1215,12 @@ class Containerize():
return return
anyLogs = False anyLogs = False
for state in deployStatusLogs.split('\n'): logging.debug('Working dir is now . (ie ci-scripts)')
myCmd.cwd = '.'
myCmd.command(f'mkdir -p {logPath}')
logging.debug(f'Working dir is now {logPath}')
myCmd.cwd = logPath
for state in deployStatusLogs.stdout.split('\n'):
res = re.search('Name|NAME|----------', state) res = re.search('Name|NAME|----------', state)
if res is not None: if res is not None:
continue continue
...@@ -1234,33 +1230,26 @@ class Containerize(): ...@@ -1234,33 +1230,26 @@ class Containerize():
if res is not None: if res is not None:
anyLogs = True anyLogs = True
cName = res.group('container_name') cName = res.group('container_name')
cmd = 'cd ' + self.yamlPath[0] + ' && docker logs ' + cName + ' > ' + cName + '.log 2>&1' cmd = 'docker logs ' + cName + ' > ' + cName + '.log 2>&1'
self.run_cmd_local_host(cmd, 30) myCmd.run(cmd, timeout=30, reportNonZero=False)
if re.search('magma-mme', cName) is not None: if re.search('magma-mme', cName) is not None:
cmd = 'cd ' + self.yamlPath[0] + ' && docker cp -L ' + cName + ':/var/log/mme.log ' + cName + '-full.log' cmd = 'docker cp -L ' + cName + ':/var/log/mme.log ' + cName + '-full.log'
self.run_cmd_local_host(cmd, 30) myCmd.run(cmd, timeout=30, reportNonZero=False)
fullStatus = True fullStatus = True
if anyLogs: if anyLogs:
cmd = 'mkdir -p '+ logPath + ' && cp ' + self.yamlPath[0] + '/*.log ' + logPath
deployStatus = self.run_cmd_local_host(cmd)
# Analyzing log file(s)! # Analyzing log file(s)!
listOfPossibleRanContainers = ['enb', 'gnb', 'cu', 'du'] listOfPossibleRanContainers = ['enb', 'gnb', 'cu', 'du']
for container in listOfPossibleRanContainers: for container in listOfPossibleRanContainers:
filenames = self.yamlPath[0] + '/*-oai-' + container + '.log' filenames = './*-oai-' + container + '.log'
cmd = 'ls ' + filenames cmd = 'ls ' + filenames
containerStatus = True lsStatus = myCmd.run(cmd, silent=True, reportNonZero=False)
try: if lsStatus.returncode != 0:
lsStatus = self.run_cmd_local_host(cmd, silent=True)
filenames = str(lsStatus).strip()
except:
containerStatus = False
if not containerStatus:
continue continue
filenames = str(lsStatus.stdout).strip()
for filename in filenames.split('\n'): for filename in filenames.split('\n'):
logging.debug('\u001B[1m Analyzing xNB logfile ' + filename + ' \u001B[0m') logging.debug('\u001B[1m Analyzing xNB logfile ' + filename + ' \u001B[0m')
logStatus = RAN.AnalyzeLogFile_eNB(filename, HTML, self.ran_checkers) logStatus = RAN.AnalyzeLogFile_eNB(f'{logPath}/{filename}', HTML, self.ran_checkers)
if (logStatus < 0): if (logStatus < 0):
fullStatus = False fullStatus = False
self.exitStatus = 1 self.exitStatus = 1
...@@ -1270,42 +1259,40 @@ class Containerize(): ...@@ -1270,42 +1259,40 @@ class Containerize():
listOfPossibleUeContainers = ['lte-ue*', 'nr-ue*'] listOfPossibleUeContainers = ['lte-ue*', 'nr-ue*']
for container in listOfPossibleUeContainers: for container in listOfPossibleUeContainers:
filenames = self.yamlPath[0] + '/*-oai-' + container + '.log' filenames = './*-oai-' + container + '.log'
cmd = 'ls ' + filenames cmd = 'ls ' + filenames
containerStatus = True containerStatus = True
try: lsStatus = myCmd.run(cmd, silent=True, reportNonZero=False)
lsStatus = self.run_cmd_local_host(cmd, silent=True) if lsStatus.returncode != 0:
filenames = str(lsStatus).strip()
except:
containerStatus = False
if not containerStatus:
continue continue
filenames = str(lsStatus.stdout).strip()
for filename in filenames.split('\n'): for filename in filenames.split('\n'):
logging.debug('\u001B[1m Analyzing UE logfile ' + filename + ' \u001B[0m') logging.debug('\u001B[1m Analyzing UE logfile ' + filename + ' \u001B[0m')
logStatus = UE.AnalyzeLogFile_UE(filename, HTML, RAN) logStatus = UE.AnalyzeLogFile_UE(f'{logPath}/{filename}', HTML, RAN)
if (logStatus < 0): if (logStatus < 0):
fullStatus = False fullStatus = False
HTML.CreateHtmlTestRow('UE log Analysis', 'KO', logStatus) HTML.CreateHtmlTestRow('UE log Analysis', 'KO', logStatus)
else: else:
HTML.CreateHtmlTestRow('UE log Analysis', 'OK', CONST.ALL_PROCESSES_OK) HTML.CreateHtmlTestRow('UE log Analysis', 'OK', CONST.ALL_PROCESSES_OK)
cmd = 'rm ' + self.yamlPath[0] + '/*.log'
deployStatus = self.run_cmd_local_host(cmd)
if self.tsharkStarted: if self.tsharkStarted:
self.tsharkStarted = True self.tsharkStarted = True
cmd = 'killall tshark || true' cmd = 'killall tshark'
self.run_cmd_local_host(cmd) myCmd.run(cmd, reportNonZero=False)
ymlPath = self.yamlPath[0].split('/') ymlPath = self.yamlPath[0].split('/')
cmd = 'mv /tmp/capture_' + ymlPath[1] + '.pcap ' + logPath + ' || true' # The working dir is still logPath
copyStatus = self.run_cmd_local_host(cmd, 100) cmd = 'mv /tmp/capture_' + ymlPath[1] + '.pcap .'
myCmd.run(cmd, timeout=100, reportNonZero=False)
self.tsharkStarted = False self.tsharkStarted = False
logging.debug('\u001B[1m Undeploying \u001B[0m') logging.debug('\u001B[1m Undeploying \u001B[0m')
cmd = 'cd ' + self.yamlPath[0] + ' && docker-compose -f docker-compose-ci.yml down' logging.debug(f'Working dir is back {self.yamlPath[0]}')
try: myCmd.cwd = self.yamlPath[0]
deployStatus = self.run_cmd_local_host(cmd, 100) cmd = 'docker-compose -f docker-compose-ci.yml down'
except Exception as e: deployStatus = myCmd.run(cmd, timeout=100)
if deployStatus.returncode != 0:
myCmd.close()
self.exitStatus = 1 self.exitStatus = 1
logging.error('Could not undeploy') logging.error('Could not undeploy')
HTML.CreateHtmlTestRow('Could not undeploy', 'KO', CONST.ALL_PROCESSES_OK) HTML.CreateHtmlTestRow('Could not undeploy', 'KO', CONST.ALL_PROCESSES_OK)
...@@ -1314,8 +1301,9 @@ class Containerize(): ...@@ -1314,8 +1301,9 @@ class Containerize():
self.deployedContainers = [] self.deployedContainers = []
# Cleaning any created tmp volume # Cleaning any created tmp volume
cmd = 'docker volume prune --force || true' cmd = 'docker volume prune --force'
deployStatus = self.run_cmd_local_host(cmd, 100) deployStatus = myCmd.run(cmd, timeout=100, reportNonZero=False)
myCmd.close()
if fullStatus: if fullStatus:
HTML.CreateHtmlTestRow('n/a', 'OK', CONST.ALL_PROCESSES_OK) HTML.CreateHtmlTestRow('n/a', 'OK', CONST.ALL_PROCESSES_OK)
...@@ -1330,11 +1318,12 @@ class Containerize(): ...@@ -1330,11 +1318,12 @@ class Containerize():
logPath = '../cmake_targets/log/' + ymlPath[1] logPath = '../cmake_targets/log/' + ymlPath[1]
# if the containers are running, recover the logs! # if the containers are running, recover the logs!
cmd = 'cd ' + self.yamlPath[0] + ' && docker-compose -f docker-compose-ci.yml ps --all' myCmd = cls_cmd.LocalCmd(d = self.yamlPath[0])
deployStatus = self.run_cmd_local_host(cmd, 30) cmd = 'docker-compose -f docker-compose-ci.yml ps --all'
deployStatus = myCmd.run(cmd, timeout=30)
cmd = 'docker stats --no-stream --format "table {{.Container}}\t{{.CPUPerc}}\t{{.MemUsage}}\t{{.MemPerc}}" ' cmd = 'docker stats --no-stream --format "table {{.Container}}\t{{.CPUPerc}}\t{{.MemUsage}}\t{{.MemPerc}}" '
anyLogs = False anyLogs = False
for state in deployStatus.split('\n'): for state in deployStatus.stdout.split('\n'):
res = re.search('Name|NAME|----------', state) res = re.search('Name|NAME|----------', state)
if res is not None: if res is not None:
continue continue
...@@ -1346,10 +1335,11 @@ class Containerize(): ...@@ -1346,10 +1335,11 @@ class Containerize():
cmd += res.group('container_name') + ' ' cmd += res.group('container_name') + ' '
message = '' message = ''
if anyLogs: if anyLogs:
stats = self.run_cmd_local_host(cmd, 30) stats = myCmd.run(cmd, timeout=30)
for statLine in stats.split('\n'): for statLine in stats.stdout.split('\n'):
logging.debug(statLine) logging.debug(statLine)
message += statLine + '\n' message += statLine + '\n'
myCmd.close()
html_queue = SimpleQueue() html_queue = SimpleQueue()
html_cell = '<pre style="background-color:white">\n' + message + '</pre>' html_cell = '<pre style="background-color:white">\n' + message + '</pre>'
...@@ -1357,17 +1347,18 @@ class Containerize(): ...@@ -1357,17 +1347,18 @@ class Containerize():
HTML.CreateHtmlTestRowQueue(self.pingOptions, 'OK', 1, html_queue) HTML.CreateHtmlTestRowQueue(self.pingOptions, 'OK', 1, html_queue)
def PingFromContainer(self, HTML, RAN, UE): def PingFromContainer(self, HTML, RAN, UE):
myCmd = cls_cmd.LocalCmd()
self.exitStatus = 0 self.exitStatus = 0
ymlPath = self.yamlPath[0].split('/') ymlPath = self.yamlPath[0].split('/')
logPath = '../cmake_targets/log/' + ymlPath[1] logPath = '../cmake_targets/log/' + ymlPath[1]
cmd = 'mkdir -p ' + logPath cmd = 'mkdir -p ' + logPath
deployStatus = self.run_cmd_local_host(cmd, silent=True) myCmd.run(cmd, silent=True)
cmd = 'docker exec ' + self.pingContName + ' /bin/bash -c "ping ' + self.pingOptions + '" 2>&1 | tee ' + logPath + '/ping_' + HTML.testCase_id + '.log || true' cmd = 'docker exec ' + self.pingContName + ' /bin/bash -c "ping ' + self.pingOptions + '" 2>&1 | tee ' + logPath + '/ping_' + HTML.testCase_id + '.log'
pingStatus = myCmd.run(cmd, timeout=100, reportNonZero=False)
myCmd.close()
deployStatus = self.run_cmd_local_host(cmd, 100) result = re.search(', (?P<packetloss>[0-9\.]+)% packet loss, time [0-9\.]+ms', pingStatus.stdout)
result = re.search(', (?P<packetloss>[0-9\.]+)% packet loss, time [0-9\.]+ms', deployStatus)
if result is None: if result is None:
self.PingExit(HTML, RAN, UE, False, 'Packet Loss Not Found') self.PingExit(HTML, RAN, UE, False, 'Packet Loss Not Found')
return return
...@@ -1377,7 +1368,7 @@ class Containerize(): ...@@ -1377,7 +1368,7 @@ class Containerize():
self.PingExit(HTML, RAN, UE, False, 'Packet Loss is 100%') self.PingExit(HTML, RAN, UE, False, 'Packet Loss is 100%')
return return
result = re.search('rtt min\/avg\/max\/mdev = (?P<rtt_min>[0-9\.]+)\/(?P<rtt_avg>[0-9\.]+)\/(?P<rtt_max>[0-9\.]+)\/[0-9\.]+ ms', deployStatus) result = re.search('rtt min\/avg\/max\/mdev = (?P<rtt_min>[0-9\.]+)\/(?P<rtt_avg>[0-9\.]+)\/(?P<rtt_max>[0-9\.]+)\/[0-9\.]+ ms', pingStatus.stdout)
if result is None: if result is None:
self.PingExit(HTML, RAN, UE, False, 'Ping RTT_Min RTT_Avg RTT_Max Not Found!') self.PingExit(HTML, RAN, UE, False, 'Ping RTT_Min RTT_Avg RTT_Max Not Found!')
return return
...@@ -1430,35 +1421,37 @@ class Containerize(): ...@@ -1430,35 +1421,37 @@ class Containerize():
self.exitStatus = 1 self.exitStatus = 1
def IperfFromContainer(self, HTML, RAN, UE): def IperfFromContainer(self, HTML, RAN, UE):
myCmd = cls_cmd.LocalCmd()
self.exitStatus = 0 self.exitStatus = 0
ymlPath = self.yamlPath[0].split('/') ymlPath = self.yamlPath[0].split('/')
logPath = '../cmake_targets/log/' + ymlPath[1] logPath = '../cmake_targets/log/' + ymlPath[1]
cmd = 'mkdir -p ' + logPath cmd = 'mkdir -p ' + logPath
logStatus = self.run_cmd_local_host(cmd, silent=True) myCmd.run(cmd, silent=True)
# Start the server process # Start the server process
cmd = f'docker exec -d {self.svrContName} /bin/bash -c "nohup iperf {self.svrOptions} > /tmp/iperf_server.log 2>&1"' cmd = f'docker exec -d {self.svrContName} /bin/bash -c "nohup iperf {self.svrOptions} > /tmp/iperf_server.log 2>&1"'
self.run_cmd_local_host(cmd) myCmd.run(cmd, reportNonZero=False)
time.sleep(3) time.sleep(3)
# Start the client process # Start the client process
cmd = f'docker exec {self.cliContName} /bin/bash -c "iperf {self.cliOptions}" 2>&1 | tee {logPath}/iperf_client_{HTML.testCase_id}.log' cmd = f'docker exec {self.cliContName} /bin/bash -c "iperf {self.cliOptions}" 2>&1 | tee {logPath}/iperf_client_{HTML.testCase_id}.log'
clientStatus = self.run_cmd_local_host(cmd, 100) clientStatus = myCmd.run(cmd, timeout=100)
# Stop the server process # Stop the server process
cmd = f'docker exec {self.svrContName} /bin/bash -c "pkill iperf"' cmd = f'docker exec {self.svrContName} /bin/bash -c "pkill iperf"'
self.run_cmd_local_host(cmd) myCmd.run(cmd)
time.sleep(3) time.sleep(3)
serverStatusFilename = f'{logPath}/iperf_server_{HTML.testCase_id}.log' serverStatusFilename = f'{logPath}/iperf_server_{HTML.testCase_id}.log'
cmd = f'docker cp {self.svrContName}:/tmp/iperf_server.log {serverStatusFilename}' cmd = f'docker cp {self.svrContName}:/tmp/iperf_server.log {serverStatusFilename}'
self.run_cmd_local_host(cmd, 30) myCmd.run(cmd, timeout=60)
myCmd.close()
# clientStatus was retrieved above. The serverStatus was # clientStatus was retrieved above. The serverStatus was
# written in the background, then copied to the local machine # written in the background, then copied to the local machine
with open(serverStatusFilename, 'r') as f: with open(serverStatusFilename, 'r') as f:
serverStatus = f.read() serverStatus = f.read()
(iperfStatus, msg) = AnalyzeIperf(self.cliOptions, clientStatus, serverStatus) (iperfStatus, msg) = AnalyzeIperf(self.cliOptions, clientStatus.stdout, serverStatus)
if iperfStatus: if iperfStatus:
logging.info('\u001B[1m Iperf Test PASS\u001B[0m') logging.info('\u001B[1m Iperf Test PASS\u001B[0m')
else: else:
...@@ -1594,13 +1587,3 @@ class Containerize(): ...@@ -1594,13 +1587,3 @@ class Containerize():
if result is None: if result is None:
mySSH.command('echo ' + password + ' | sudo -S iptables -P FORWARD ACCEPT', '\$', 10) mySSH.command('echo ' + password + ' | sudo -S iptables -P FORWARD ACCEPT', '\$', 10)
mySSH.close() mySSH.close()
def run_cmd_local_host(self, lCmd, lTimeout=10, silent=False):
if not silent:
logging.info(lCmd)
result = subprocess.run(lCmd, shell=True, check=True,
universal_newlines=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
timeout=lTimeout)
return str(result.stdout)
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment