Merge remote-tracking branch 'origin/ci-improvements' into integration_2022_wk03_b

ci-scripts/cls_containerize.py

@@ -221,7 +221,7 @@ class Containerize():
 		if (self.ranAllowMerge):
 			imageTag = 'ci-temp'
 			if self.ranTargetBranch == 'develop':
-				mySSH.command('git diff HEAD..origin/develop -- docker/Dockerfile.base' + self.dockerfileprefix + ' | grep --colour=never -i INDEX', '\$', 5)
+				mySSH.command('git diff HEAD..origin/develop -- cmake_targets/build_oai cmake_targets/tools/build_helper docker/Dockerfile.base' + self.dockerfileprefix + ' | grep --colour=never -i INDEX', '\$', 5)
 				result = re.search('index', mySSH.getBefore())
 				if result is not None:
 					forceBaseImageBuild = True
@@ -341,7 +341,7 @@ class Containerize():
 		# Remove all intermediate build images
 		if self.ranAllowMerge and forceBaseImageBuild:
 			mySSH.command(self.cli + ' image rm ' + baseImage + ':' + baseTag + ' || true', '\$', 30)
-		mySSH.command(self.cli + ' image rm ran-build:' + imageTag + ' || true','\$', 5)
+		mySSH.command(self.cli + ' image rm ran-build:' + imageTag + ' || true','\$', 30)
 		# Cleaning any created tmp volume
 		mySSH.command(self.cli + ' volume prune --force || true','\$', 15)
 		mySSH.close()
@@ -586,11 +586,20 @@ class Containerize():
 		mySSH.command('cd ' + lSourcePath + '/' + self.yamlPath[self.eNB_instance], '\$', 5)
 		# Currently support only one
 		mySSH.command('docker-compose --file ci-docker-compose.yml config', '\$', 5)
+		containerName = ''
+		containerToKill = False
 		result = re.search('container_name: (?P<container_name>[a-zA-Z0-9\-\_]+)', mySSH.getBefore())
 		if self.eNB_logFile[self.eNB_instance] == '':
 			self.eNB_logFile[self.eNB_instance] = 'enb_' + HTML.testCase_id + '.log'
 		if result is not None:
 			containerName = result.group('container_name')
+			containerToKill = True
+		if containerToKill:
+			mySSH.command('docker inspect ' + containerName, '\$', 30)
+			result = re.search('Error: No such object: ' + containerName, mySSH.getBefore())
+			if result is not None:
+				containerToKill = False
+		if containerToKill:
 			mySSH.command('docker kill --signal INT ' + containerName, '\$', 30)
 			time.sleep(5)
 			mySSH.command('docker logs ' + containerName + ' > ' + lSourcePath + '/cmake_targets/' + self.eNB_logFile[self.eNB_instance], '\$', 30)
@@ -603,20 +612,26 @@ class Containerize():
 		mySSH.close()
 
 		# Analyzing log file!
-		copyin_res = mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/' + self.eNB_logFile[self.eNB_instance], '.')
+		if containerToKill:
+			copyin_res = mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/' + self.eNB_logFile[self.eNB_instance], '.')
+		else:
+			copyin_res = 0
 		nodeB_prefix = 'e'
 		if (copyin_res == -1):
 			HTML.htmleNBFailureMsg='Could not copy ' + nodeB_prefix + 'NB logfile to analyze it!'
 			HTML.CreateHtmlTestRow('N/A', 'KO', CONST.ENB_PROCESS_NOLOGFILE_TO_ANALYZE)
 		else:
-			logging.debug('\u001B[1m Analyzing ' + nodeB_prefix + 'NB logfile \u001B[0m ' + self.eNB_logFile[self.eNB_instance])
-			logStatus = RAN.AnalyzeLogFile_eNB(self.eNB_logFile[self.eNB_instance], HTML)
+			if containerToKill:
+				logging.debug('\u001B[1m Analyzing ' + nodeB_prefix + 'NB logfile \u001B[0m ' + self.eNB_logFile[self.eNB_instance])
+				logStatus = RAN.AnalyzeLogFile_eNB(self.eNB_logFile[self.eNB_instance], HTML)
+			else:
+				logStatus = 0
 			if (logStatus < 0):
 				HTML.CreateHtmlTestRow(RAN.runtime_stats, 'KO', logStatus)
 			else:
 				HTML.CreateHtmlTestRow(RAN.runtime_stats, 'OK', CONST.ALL_PROCESSES_OK)
 			# all the xNB run logs shall be on the server 0 for logCollecting
-			if self.eNB_serverId[self.eNB_instance] != '0':
+			if containerToKill and self.eNB_serverId[self.eNB_instance] != '0':
 				mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, './' + self.eNB_logFile[self.eNB_instance], self.eNBSourceCodePath + '/cmake_targets/')
 		logging.info('\u001B[1m Undeploying OAI Object Pass\u001B[0m')
 
@@ -689,7 +704,7 @@ class Containerize():
 	def CaptureOnDockerNetworks(self):
 		cmd = 'cd ' + self.yamlPath[0] + ' && docker-compose -f docker-compose-ci.yml config | grep com.docker.network.bridge.name | sed -e "s@^.*name: @@"'
 		networkNames = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=10)
-		cmd = 'sudo nohup tshark -f "not tcp and not arp and not port 53 and not host archive.ubuntu.com and not host security.ubuntu.com"'
+		cmd = 'sudo nohup tshark -f "not tcp and not arp and not port 53 and not host archive.ubuntu.com and not host security.ubuntu.com and not port 2152"'
 		for name in networkNames.split('\n'):
 			res = re.search('rfsim', name)
 			if res is not None:
@@ -700,7 +715,7 @@ class Containerize():
 		logging.debug(cmd)
 		networkNames = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=10)
 
-	def UndeployGenObject(self, HTML):
+	def UndeployGenObject(self, HTML, RAN):
 		self.exitStatus = 0
 		ymlPath = self.yamlPath[0].split('/')
 		logPath = '../cmake_targets/log/' + ymlPath[1]
@@ -730,8 +745,36 @@ class Containerize():
 				cmd = 'cd ' + self.yamlPath[0] + ' && docker logs ' + cName + ' > ' + cName + '.log 2>&1'
 				logging.debug(cmd)
 				deployStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=30)
+		fullStatus = True
 		if anyLogs:
-			cmd = 'mkdir -p '+ logPath + ' && mv ' + self.yamlPath[0] + '/*.log ' + logPath
+			cmd = 'mkdir -p '+ logPath + ' && cp ' + self.yamlPath[0] + '/*.log ' + logPath
+			logging.debug(cmd)
+			deployStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=10)
+
+			# Analyzing log file!
+			listOfPossibleRanContainers = ['enb', 'gnb', 'cu', 'du']
+			for container in listOfPossibleRanContainers:
+				filename = self.yamlPath[0] + '/rfsim?g-oai-' + container + '.log'
+				cmd = 'ls ' + filename
+				containerStatus = True
+				try:
+					lsStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=10)
+					filename = str(lsStatus).strip()
+				except:
+					containerStatus = False
+				if not containerStatus:
+					continue
+
+				logging.debug('\u001B[1m Analyzing xNB logfile ' + filename + ' \u001B[0m')
+				# For the moment just assume this exists
+				logStatus = RAN.AnalyzeLogFile_eNB(filename, HTML)
+				if (logStatus < 0):
+					fullStatus = False
+					HTML.CreateHtmlTestRow(RAN.runtime_stats, 'KO', logStatus)
+				else:
+					HTML.CreateHtmlTestRow(RAN.runtime_stats, 'OK', CONST.ALL_PROCESSES_OK)
+
+			cmd = 'rm ' + self.yamlPath[0] + '/*.log'
 			logging.debug(cmd)
 			deployStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=10)
 			if self.tsharkStarted:
@@ -747,6 +790,7 @@ class Containerize():
 				logging.debug(cmd)
 				copyStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=10)
 
+		logging.debug('\u001B[1m Undeploying \u001B[0m')
 		cmd = 'cd ' + self.yamlPath[0] + ' && docker-compose -f docker-compose-ci.yml down'
 		logging.debug(cmd)
 		try:
@@ -763,8 +807,12 @@ class Containerize():
 		logging.debug(cmd)
 		deployStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=100)
 
-		HTML.CreateHtmlTestRow('n/a', 'OK', CONST.ALL_PROCESSES_OK)
-		logging.info('\u001B[1m Undeploying OAI Object(s) PASS\u001B[0m')
+		if fullStatus:
+			HTML.CreateHtmlTestRow('n/a', 'OK', CONST.ALL_PROCESSES_OK)
+			logging.info('\u001B[1m Undeploying OAI Object(s) PASS\u001B[0m')
+		else:
+			HTML.CreateHtmlTestRow('n/a', 'KO', CONST.ALL_PROCESSES_OK)
+			logging.info('\u001B[1m Undeploying OAI Object(s) FAIL\u001B[0m')
 
 	def PingFromContainer(self, HTML):
 		self.exitStatus = 0

ci-scripts/epc.py

@@ -133,7 +133,7 @@ class EPCManagement():
 		mySSH.open(self.IPAddress, self.UserName, self.Password)
 		if re.match('OAI-Rel14-Docker', self.Type, re.IGNORECASE):
 			logging.debug('Using the OAI EPC Release 14 MME in Docker')
-			mySSH.command('docker exec -d ' + self.containerPrefix + '-oai-mme /bin/bash -c "nohup tshark -i eth0 -i lo:s10 -w /tmp/mme_check_run.pcap 2>&1 > /dev/null"', '\$', 5)
+			mySSH.command('docker exec -d ' + self.containerPrefix + '-oai-mme /bin/bash -c "nohup tshark -i eth0 -i lo:s10 -f "not port 2152" -w /tmp/mme_check_run.pcap 2>&1 > /dev/null"', '\$', 5)
 			time.sleep(5)
 			mySSH.command('docker exec -d ' + self.containerPrefix + '-oai-mme /bin/bash -c "nohup ./bin/oai_mme -c ./etc/' + self.mmeConfFile + ' > mme_check_run.log 2>&1"', '\$', 5)
 		elif re.match('OAI-Rel14-CUPS', self.Type, re.IGNORECASE):
@@ -198,8 +198,8 @@ class EPCManagement():
 		mySSH.open(self.IPAddress, self.UserName, self.Password)
 		if re.match('OAI-Rel14-Docker', self.Type, re.IGNORECASE):
 			logging.debug('Using the OAI EPC Release 14 SPGW-CUPS in Docker')
-			mySSH.command('docker exec -d ' + self.containerPrefix + '-oai-spgwc /bin/bash -c "nohup tshark -i eth0 -i lo:p5c -i lo:s5c -w /tmp/spgwc_check_run.pcap 2>&1 > /dev/null"', '\$', 5)
-			mySSH.command('docker exec -d ' + self.containerPrefix + '-oai-spgwu-tiny /bin/bash -c "nohup tshark -i eth0 -w /tmp/spgwu_check_run.pcap 2>&1 > /dev/null"', '\$', 5)
+			mySSH.command('docker exec -d ' + self.containerPrefix + '-oai-spgwc /bin/bash -c "nohup tshark -i eth0 -i lo:p5c -i lo:s5c -f "not port 2152" -w /tmp/spgwc_check_run.pcap 2>&1 > /dev/null"', '\$', 5)
+			mySSH.command('docker exec -d ' + self.containerPrefix + '-oai-spgwu-tiny /bin/bash -c "nohup tshark -i eth0 -f "not port 2152" -w /tmp/spgwu_check_run.pcap 2>&1 > /dev/null"', '\$', 5)
 			time.sleep(5)
 			mySSH.command('docker exec -d ' + self.containerPrefix + '-oai-spgwc /bin/bash -c "nohup ./bin/oai_spgwc -o -c ./etc/spgw_c.conf > spgwc_check_run.log 2>&1"', '\$', 5)
 			time.sleep(5)
@@ -532,6 +532,7 @@ class EPCManagement():
 			logging.debug('Terminating OAI CN5G')
 			mySSH.command('cd /opt/oai-cn5g-fed/docker-compose', '\$', 5)
 			mySSH.command('./core-network.sh stop nrf spgwu', '\$', 60)
+			mySSH.command('docker volume prune --force || true', '\$', 60)
 			time.sleep(2)
 			mySSH.command('tshark -r /tmp/oai-cn5g.pcap | egrep --colour=never "Tracking area update" ','\$', 30)
 			result = re.search('Tracking area update request', mySSH.getBefore())
@@ -760,6 +761,7 @@ class EPCManagement():
 			nbContainers += 1
 
 		mySSH.command('docker-compose down', '\$', 60)
+		mySSH.command('docker volume prune --force || true', '\$', 60)
 		mySSH.command('docker inspect --format=\'{{.State.Health.Status}}\' ' + listOfContainers, '\$', 10)
 		noMoreContainerNb = mySSH.getBefore().count('No such object')
 		mySSH.command('docker inspect --format=\'{{.Name}}\' prod-oai-public-net prod-oai-private-net', '\$', 10)

ci-scripts/main.py

@@ -909,7 +909,7 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re
 					if CONTAINERS.exitStatus==1:
 						RAN.prematureExit = True
 				elif action == 'UndeployGenObject':
-					CONTAINERS.UndeployGenObject(HTML)
+					CONTAINERS.UndeployGenObject(HTML, RAN)
 					if CONTAINERS.exitStatus==1:
 						RAN.prematureExit = True
 				elif action == 'PingFromContainer':

ci-scripts/ran.py

@@ -159,6 +159,7 @@ class RANManagement():
 		# Raphael: here add a check if git clone or git fetch went smoothly
 		mySSH.command('git config user.email "jenkins@openairinterface.org"', '\$', 5)
 		mySSH.command('git config user.name "OAI Jenkins"', '\$', 5)
+		mySSH.command('git advice.detachedHead false', '\$', 5)
 		# Checking the BUILD INFO file
 		if not self.backgroundBuild:
 			mySSH.command('ls *.txt', '\$', 5)
@@ -203,6 +204,7 @@ class RANManagement():
 			else:
 				logging.debug('Merging with the target branch: ' + self.ranTargetBranch)
 				mySSH.command('git merge --ff origin/' + self.ranTargetBranch + ' -m "Temporary merge for CI"', '\$', 5)
+		logging.debug(mySSH.getBefore()) # print what git said when merging/checking out
 		mySSH.command('source oaienv', '\$', 5)
 		mySSH.command('cd cmake_targets', '\$', 5)
 		mySSH.command('mkdir -p log', '\$', 5)
@@ -212,6 +214,7 @@ class RANManagement():
 			mySSH.command('echo "./build_oai ' + self.Build_eNB_args + '" > ./my-lte-softmodem-build.sh', '\$', 5)
 			mySSH.command('chmod 775 ./my-lte-softmodem-build.sh', '\$', 5)
 			mySSH.command('echo ' + lPassWord + ' | sudo -S ls', '\$', 5)
+			logging.debug(mySSH.getBefore()) # print current directory contents for verification
 			mySSH.command('echo $USER; nohup sudo -E ./my-lte-softmodem-build.sh' + ' > ' + lSourcePath + '/cmake_targets/compile_oai_enb.log ' + ' 2>&1 &', lUserName, 5)
 			mySSH.close()
 			HTML.CreateHtmlTestRow(self.Build_eNB_args, 'OK', CONST.ALL_PROCESSES_OK)
@@ -371,10 +374,11 @@ class RANManagement():
 			result = re.search('interfaceToUse=(?P<eth_interface>[a-zA-Z0-9\-\_]+)done', mySSH.getBefore())
 			if result is not None:
 				eth_interface = result.group('eth_interface')
-				logging.debug('\u001B[1m Launching tshark on interface ' + eth_interface + '\u001B[0m')
+				fltr = 'port 38412 or port 36412 or port 36422' # NGAP, S1AP, X2AP
+				logging.debug('\u001B[1m Launching tshark on interface ' + eth_interface + ' with filter "' + fltr + '"\u001B[0m')
 				pcapfile = pcapfile_prefix + self.testCase_id + '_log.pcap'
 				mySSH.command('echo ' + lPassWord + ' | sudo -S rm -f /tmp/' + pcapfile , '\$', 5)
-				mySSH.command('echo $USER; nohup sudo -E tshark  -i ' + eth_interface + ' -w /tmp/' + pcapfile + ' > /dev/null 2>&1 &','\$', 5)
+				mySSH.command('echo $USER; nohup sudo -E tshark  -i ' + eth_interface + ' -f "' + fltr + '" -w /tmp/' + pcapfile + ' > /dev/null 2>&1 &','\$', 5)
 			mySSH.close()
 			
 
@@ -390,10 +394,11 @@ class RANManagement():
 			result = re.search('interfaceToUse=(?P<eth_interface>[a-zA-Z0-9\-\_]+)done', mySSH.getBefore())
 			if result is not None:
 				eth_interface = result.group('eth_interface')
-				logging.debug('\u001B[1m Launching tshark on interface ' + eth_interface + '\u001B[0m')
+				fltr = 'port 38412 or port 36412 or port 36422' # NGAP, S1AP, X2AP
+				logging.debug('\u001B[1m Launching tshark on interface ' + eth_interface + ' with filter "' + fltr + '"\u001B[0m')
 				self.epcPcapFile = 'enb_' + self.testCase_id + '_s1log.pcap'
 				mySSH.command('echo ' + localEpcPassword + ' | sudo -S rm -f /tmp/' + self.epcPcapFile , '\$', 5)
-				mySSH.command('echo $USER; nohup sudo tshark -f "host ' + lIpAddr +'" -i ' + eth_interface + ' -w /tmp/' + self.epcPcapFile + ' > /tmp/tshark.log 2>&1 &', localEpcUserName, 5)
+				mySSH.command('echo $USER; nohup sudo tshark -f "host ' + lIpAddr +'" -i ' + eth_interface + ' -f "' + fltr + '" -w /tmp/' + self.epcPcapFile + ' > /tmp/tshark.log 2>&1 &', localEpcUserName, 5)
 			mySSH.close()
 		mySSH.open(lIpAddr, lUserName, lPassWord)
 		mySSH.command('cd ' + lSourcePath, '\$', 5)
@@ -738,7 +743,10 @@ class RANManagement():
 		mySSH.command('echo ' + self.eNBPassword + ' | sudo -S mv /tmp/enb_*.pcap .','\$',20)
 		mySSH.command('echo ' + self.eNBPassword + ' | sudo -S mv /tmp/gnb_*.pcap .','\$',20)
 		mySSH.command('echo ' + self.eNBPassword + ' | sudo -S rm -f enb.log.zip', '\$', 5)
-		mySSH.command('echo ' + self.eNBPassword + ' | sudo -S zip enb.log.zip enb*.log core* enb_*record.raw enb_*.pcap gnb_*.pcap enb_*txt physim_*.log *stats.log *monitor.pickle *monitor*.png ping*.log.png log/*/*.log log/*/*.pcap', '\$', 60)
+		mySSH.command('echo ' + self.eNBPassword + ' | sudo -S zip enb.log.zip enb*.log enb_*record.raw enb_*.pcap gnb_*.pcap enb_*txt physim_*.log *stats.log *monitor.pickle *monitor*.png ping*.log.png log/*/*.log log/*/*.pcap', '\$', 60)
+		result = re.search('core.\d+', mySSH.getBefore())
+		if result is not None:
+			mySSH.command('echo ' + self.eNBPassword + ' | sudo -S zip enb.log.zip core* ran_build/build/{lte,nr}-softmodem', '\$', 60) # add core and executable to zip
 		mySSH.command('echo ' + self.eNBPassword + ' | sudo -S rm enb*.log core* enb_*record.raw enb_*.pcap gnb_*.pcap enb_*txt physim_*.log *stats.log *monitor.pickle *monitor*.png ping*.log.png log/*/*.log log/*/*.pcap', '\$', 15)
 		mySSH.close()
 
@@ -1016,12 +1024,9 @@ class RANManagement():
 			datalog_rt_stats = yaml.load(f,Loader=yaml.FullLoader)
 		rt_keys = datalog_rt_stats['Ref'] #we use the keys from the Ref field  
 
-		if (os.path.isfile('./nrL1_stats.log')) and (os.path.isfile('./nrL1_stats.log')):
-			stat_files_present=True
-		else:
-			stat_files_present=False
-			logging.debug("NR Stats files for RT analysis not found")
-		if stat_files_present:
+		if os.path.isfile('./nrL1_stats.log') and os.path.isfile('./nrMAC_stats.log'):
+			# don't use CI-nrL1_stats.log, as this will increase the processing time for
+			# no reason, we just need the last occurence
 			nrL1_stats = open('./nrL1_stats.log', 'r')
 			nrMAC_stats = open('./nrMAC_stats.log', 'r')
 			for line in nrL1_stats.readlines():
@@ -1042,6 +1047,8 @@ class RANManagement():
 							real_time_stats[k]=tmp.group(1)
 			nrL1_stats.close()
 			nrMAC_stats.close()
+		else:
+			logging.debug("NR Stats files for RT analysis not found")
 
 		#stdout log file and stat log files analysis completed
 		logging.debug('   File analysis (stdout, stats) completed')

ci-scripts/sshconnection.py

@@ -157,22 +157,24 @@ class SSHConnection():
 		if not silent:
 			logging.debug(commandline)
 		self.cmd2Results = ''
+		noHistoryCmd = 'unset HISTFILE; ' + commandline
 		myHost = self.username + '@' + self.ipaddress
 		# CAUTION: THIS METHOD IMPLIES THAT THERE ARE VALID SSH KEYS
 		# BETWEEN THE PYTHON EXECUTOR NODE AND THE REMOTE HOST
 		# OTHERWISE IT WON'T WORK
-		lSsh = subprocess.Popen(["ssh", "%s" % myHost, commandline],shell=False,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
+		lSsh = subprocess.Popen(["ssh", "%s" % myHost, noHistoryCmd],shell=False,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
 		self.cmd2Results = str(lSsh.stdout.readlines())
 
 	def command3(self, commandline, timeout, silent=False):
 		if not silent:
 			logging.debug(commandline)
 		self.cmd2Results = ''
+		noHistoryCmd = 'unset HISTFILE; ' + commandline
 		myHost = self.username + '@' + self.ipaddress
 		# CAUTION: THIS METHOD IMPLIES THAT THERE ARE VALID SSH KEYS
 		# BETWEEN THE PYTHON EXECUTOR NODE AND THE REMOTE HOST
 		# OTHERWISE IT WON'T WORK
-		lSsh = subprocess.Popen(["ssh", "%s" % myHost, commandline],shell=False,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
+		lSsh = subprocess.Popen(["ssh", "%s" % myHost, noHistoryCmd],shell=False,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
 		return lSsh.stdout.readlines()
 
 		

ci-scripts/stats_monitor.py

@@ -61,9 +61,16 @@ class StatMonitor():
 
     def collect(self,node_type):
         if node_type=='enb':
-            cmd='cat L1_stats.log MAC_stats.log PDCP_stats.log RRC_stats.log'
+            files = ["L1_stats.log", "MAC_stats.log", "PDCP_stats.log", "RRC_stats.log"]
         else: #'gnb'
-            cmd='cat nrL1_stats.log nrMAC_stats.log nrPDCP_stats.log nrRRC_stats.log'
+            files = ["nrL1_stats.log", "nrMAC_stats.log", "nrPDCP_stats.log", "nrRRC_stats.log"]
+        # append each file's contents to another file (prepended with CI-) for
+        # post-mortem/debugging analysis
+        for f in files:
+            cmd = rf'cat {f} >> CI-{f}'
+            subprocess.Popen(shlex.split(cmd))
+        # join the files for further processing
+        cmd = rf'cat {shlex.join(files)}'
         process=subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE)
         output = process.stdout.readlines()
         if node_type=='enb':

ci-scripts/stats_monitor.py.old

-import subprocess
-import time
-import shlex
-import re
-import sys
-import matplotlib.pyplot as plt
-import pickle
-import numpy as np
-import os
-
-def collect(d, node_type):
-    if node_type=='enb':
-        cmd='cat L1_stats.log MAC_stats.log PDCP_stats.log RRC_stats.log'
-    else: #'gnb'
-        cmd='cat nrL1_stats.log nrMAC_stats.log nrPDCP_stats.log nrRRC_stats.log'
-    process=subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE)
-    output = process.stdout.readlines()
-    for l in output:
-        tmp=l.decode("utf-8")
-        result=re.match(rf'^.*\bPHR\b ([0-9]+).+\bbler\b ([0-9]+\.[0-9]+).+\bmcsoff\b ([0-9]+).+\bmcs\b ([0-9]+)',tmp)
-        if result is not None:
-            d['PHR'].append(int(result.group(1)))
-            d['bler'].append(float(result.group(2)))
-            d['mcsoff'].append(int(result.group(3)))
-            d['mcs'].append(int(result.group(4)))
-
-
-def graph(d, node_type):
-
-
-    figure, axis = plt.subplots(4, 1,figsize=(10, 10)) 
-
-    major_ticks = np.arange(0, len(d['PHR'])+1, 1)
-    axis[0].set_xticks(major_ticks)
-    axis[0].set_xticklabels([])
-    axis[0].plot(d['PHR'],marker='o')
-    axis[0].set_xlabel('time')
-    axis[0].set_ylabel('PHR')
-    axis[0].set_title("PHR")
-  
-    major_ticks = np.arange(0, len(d['bler'])+1, 1)
-    axis[1].set_xticks(major_ticks)
-    axis[1].set_xticklabels([])
-    axis[1].plot(d['bler'],marker='o')
-    axis[1].set_xlabel('time')
-    axis[1].set_ylabel('bler')
-    axis[1].set_title("bler")
-
-    major_ticks = np.arange(0, len(d['mcsoff'])+1, 1)
-    axis[2].set_xticks(major_ticks)
-    axis[2].set_xticklabels([])
-    axis[2].plot(d['mcsoff'],marker='o')
-    axis[2].set_xlabel('time')
-    axis[2].set_ylabel('mcsoff')
-    axis[2].set_title("mcsoff")
-
-    major_ticks = np.arange(0, len(d['mcs'])+1, 1)
-    axis[3].set_xticks(major_ticks)
-    axis[3].set_xticklabels([])
-    axis[3].plot(d['mcs'],marker='o')
-    axis[3].set_xlabel('time')
-    axis[3].set_ylabel('mcs')
-    axis[3].set_title("mcs")
-
-    plt.tight_layout()
-    # Combine all the operations and display
-    plt.savefig(node_type+'_stats_monitor.png')
-    plt.show()
-
-if __name__ == "__main__":
-
-    node_type = sys.argv[1]#enb or gnb
-
-    d={}
-    d['PHR']=[]
-    d['bler']=[]
-    d['mcsoff']=[]
-    d['mcs']=[]
-
-
-    cmd='ps aux | grep modem | grep -v grep'
-    process=subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
-    output = process.stdout.readlines()
-    while len(output)!=0 :
-        collect(d, node_type)
-        process=subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
-        output = process.stdout.readlines()
-        time.sleep(1)
-    print('process stopped')
-    with open(node_type+'_stats_monitor.pickle', 'wb') as handle:
-        pickle.dump(d, handle, protocol=pickle.HIGHEST_PROTOCOL)
-    graph(d, node_type)
-
-

ci-scripts/stats_monitor_dev.py

-"""
-To create graphs and pickle from runtime statistics in L1,MAC,RRC,PDCP files
-"""
-
-import subprocess
-import time
-import shlex
-import re
-import sys
-import pickle
-import matplotlib.pyplot as plt
-import numpy as np
-import yaml
-
-
-class StatMonitor():
-    def __init__(self,):
-        with open('stats_monitor_conf.yaml','r') as file:
-            self.d = yaml.load(file)
-        for node in self.d:
-            for metric in self.d[node]:
-                self.d[node][metric]=[]
-
-
-    def process_gnb (self,node_type,output):
-        for line in output:
-            tmp=line.decode("utf-8")
-            result=re.match(r'^.*\bdlsch_rounds\b ([0-9]+)\/([0-9]+).*\bdlsch_errors\b ([0-9]+)',tmp)
-            if result is not None:
-                self.d[node_type]['dlsch_err'].append(int(result.group(3)))
-                percentage=float(result.group(2))/float(result.group(1))
-                self.d[node_type]['dlsch_err_perc_round_1'].append(percentage)
-            result=re.match(r'^.*\bulsch_rounds\b ([0-9]+)\/([0-9]+).*\bulsch_errors\b ([0-9]+)',tmp)
-            if result is not None:
-                self.d[node_type]['ulsch_err'].append(int(result.group(3)))
-                percentage=float(result.group(2))/float(result.group(1))
-                self.d[node_type]['ulsch_err_perc_round_1'].append(percentage)
-
-
-    def process_enb (self,node_type,output):
-        for line in output:
-            tmp=line.decode("utf-8")
-            result=re.match(r'^.*\bPHR\b ([0-9]+).+\bbler\b ([0-9]+\.[0-9]+).+\bmcsoff\b ([0-9]+).+\bmcs\b ([0-9]+)',tmp)
-            if result is not None:
-                self.d[node_type]['PHR'].append(int(result.group(1)))
-                self.d[node_type]['bler'].append(float(result.group(2)))
-                self.d[node_type]['mcsoff'].append(int(result.group(3)))
-                self.d[node_type]['mcs'].append(int(result.group(4)))
-
-
-    def collect(self,node_type):
-        if node_type=='enb':
-            cmd='cat L1_stats.log MAC_stats.log PDCP_stats.log RRC_stats.log'
-        else: #'gnb'
-            cmd='cat nrL1_stats.log nrMAC_stats.log nrPDCP_stats.log nrRRC_stats.log'
-        process=subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE)
-        output = process.stdout.readlines()
-        if node_type=='enb':
-            self.process_enb(node_type,output)
-        else: #'gnb'
-            self.process_gnb(node_type,output)
-
-
-    def graph(self,node_type):
-        col = 1
-        figure, axis = plt.subplots(len(self.d[node_type]), col ,figsize=(10, 10))
-        i=0
-        for metric in self.d[node_type]:
-            major_ticks = np.arange(0, len(self.d[node_type][metric])+1, 1)
-            axis[i].set_xticks(major_ticks)
-            axis[i].set_xticklabels([])
-            axis[i].plot(self.d[node_type][metric],marker='o')
-            axis[i].set_xlabel('time')
-            axis[i].set_ylabel(metric)
-            axis[i].set_title(metric)
-            i+=1
-
-        plt.tight_layout()
-        # Combine all the operations and display
-        plt.savefig(node_type+'_stats_monitor.png')
-        plt.show()
-
-
-if __name__ == "__main__":
-
-    node = sys.argv[1]#enb or gnb
-    mon=StatMonitor()
-
-    #collecting stats when modem process is stopped
-    CMD='ps aux | grep mode | grep -v grep'
-    process=subprocess.Popen(CMD, shell=True, stdout=subprocess.PIPE)
-    output = process.stdout.readlines()
-    while len(output)!=0 :
-        mon.collect(node)
-        process=subprocess.Popen(CMD, shell=True, stdout=subprocess.PIPE)
-        output = process.stdout.readlines()
-        time.sleep(1)
-    print('Process stopped')
-    with open(node+'_stats_monitor.pickle', 'wb') as handle:
-        pickle.dump(mon.d, handle, protocol=pickle.HIGHEST_PROTOCOL)
-    mon.graph(node)

ci-scripts/xml_files/container_5g_rfsim.xml

@@ -93,7 +93,7 @@
                 <server_container_name>rfsim5g-oai-nr-ue</server_container_name>
                 <client_container_name>rfsim5g-oai-ext-dn</client_container_name>
                 <server_options>-B 12.1.1.2 -u -i 1 -s</server_options>
-                <client_options>-c 12.1.1.2 -u -i 1 -t 30 -b 400K</client_options>
+                <client_options>-c 12.1.1.2 -u -i 1 -t 30 -b 3M</client_options>
         </testCase>
 
         <testCase id="030002">
@@ -102,7 +102,7 @@
                 <server_container_name>rfsim5g-oai-ext-dn</server_container_name>
                 <client_container_name>rfsim5g-oai-nr-ue</client_container_name>
                 <server_options>-u -i 1 -s</server_options>
-                <client_options>-B 12.1.1.2 -c 192.168.72.135 -u -i 1 -t 30 -b 20K</client_options>
+                <client_options>-B 12.1.1.2 -c 192.168.72.135 -u -i 1 -t 30 -b 1M</client_options>
         </testCase>
 
         <testCase id="100001">

ci-scripts/xml_files/fr1_sa_oaiue_x300.xml

@@ -26,19 +26,19 @@
 	<htmlTabIcon>tasks</htmlTabIcon>
 	<repeatCount>1</repeatCount>
 	<TestCaseRequestedList>
- 040000
+ 041000
  000002
- 010000
+ 011000
  000002
- 050000
- 050001
+ 051000
+ 051001
  000001
  010002
  080000
 	</TestCaseRequestedList>
 	<TestCaseExclusionList></TestCaseExclusionList>
 
-	<testCase id="010000">
+	<testCase id="011000">
 		<class>Initialize_OAI_UE</class>
 		<desc>Initialize OAI UE (X300)</desc>
 		<air_interface>nr</air_interface>
@@ -51,7 +51,7 @@
 	</testCase>
 
 
-	<testCase id="040000">
+	<testCase id="041000">
 		<class>Initialize_eNB</class>
 		<desc>Initialize gNB</desc>
 		<Initialize_eNB_args>-O ci-scripts/conf_files/gnb.band78.sa.fr1.106PRB.usrpn310.conf --sa --usrp-tx-thread-config 1 --log_config.global_log_options level,nocolor,time</Initialize_eNB_args>
@@ -76,14 +76,14 @@
 	</testCase>
 
 
-	<testCase id="050000">
+	<testCase id="051000">
 		<class>Ping</class>
 		<desc>Ping from CN to UE: 20pings in 20sec</desc>
 		<ping_args>-c 20</ping_args>
 		<ping_packetloss_threshold>5</ping_packetloss_threshold>
 	</testCase>
 
-	<testCase id="050001">
+	<testCase id="051001">
 		<class>Ping</class>
 		<desc>Ping from CN to UE: 100pings in 20sec</desc>
 		<ping_args>-c 100 -i 0.2</ping_args>

cmake_targets/build_oai

@@ -99,25 +99,23 @@ Options
   Makes the LTE softmodem
 --eNBocp
   Makes the OCP LTE softmodem
--gNB
+--gNB
   Makes the NR softmodem
---nrUE
-  Makes the NR UE softmodem
 --RU
   Makes the OAI RRU
 --UE
    Makes the UE specific parts (ue_ip, usim, nvram) from the given configuration file
+--nrUE
+  Makes the NR UE softmodem
 --UE-conf-nvram [configuration file]
    Specify conf_nvram_path (default \"$conf_nvram_path\")
 --UE-gen-nvram [output path]
    Specify gen_nvram_path (default \"$gen_nvram_path\")
---HWLAT
-    Makes test program for haw latency tests
--a | --agent
-   Enables agent for software-defined control of the eNB
 -w | --hardware
    EXMIMO, USRP, BLADERF, LMSSDR, IRIS, ADRV9371_ZC706, SIMU, AW2SORI, None (Default)
    Adds this RF board support (in external packages installation and in compilation)
+-t | --transport
+   Selects the transport protocol type, options: None, Ethernet, benetel4g, benetel5g
 -P | --phy_simulators
    Makes the unitary tests Layer 1 simulators
 -S | --core_simulators
@@ -128,6 +126,14 @@ Options
    runs only specified test cases specified here. This flag is only valid with -s
 -V | --vcd
    Adds a debgging facility to the binary files: GUI with major internal synchronization events
+-x | --xforms
+   Will compile with software oscilloscope features
+--HWLAT
+    Makes test program for haw latency tests
+--HWLAT_TEST
+  Compiles a hw latency test program
+--verbose-ci
+  Compile with verbose instructions in CI Docker env
 --verbose-compile
    Shows detailed compilation instructions in makefile
 --cflags_processor
@@ -163,6 +169,10 @@ Options
    Build for I/Q record-playback modes
 -k | --skip-shared-libraries
    Skip build for shared libraries to reduce compilation time when building frequently for debugging purposes
+--ninja
+  Tell cmake to use the Ninja build system. Without, will generate make files
+--sanitize-address | -fsanitize=address
+   Enable the address sanitizer on all targets
 --ittiSIM
   Makes the itti simulator
 -h | --help

cmake_targets/tools/build_helper

@@ -756,6 +756,7 @@ check_install_oai_software() {
 	libidn11-dev \
 	libmysqlclient-dev  \
 	libpython2.7-dev \
+        libreadline-dev \
 	libsctp1  \
 	libsctp-dev  \
 	libssl-dev  \
@@ -842,6 +843,7 @@ check_install_oai_software() {
       patch \
       psmisc \
       python \
+      readline-devel \
       subversion \
       xmlstarlet \
       python-pip \

docker/Dockerfile.build.rhel8.2

@@ -35,4 +35,4 @@ COPY . .
 RUN /bin/sh oaienv && \ 
     cd cmake_targets && \
     mkdir -p log && \
-    ./build_oai --eNB --gNB --RU --UE --nrUE --ninja -w USRP --verbose-ci
+    ./build_oai --eNB --gNB --RU --UE --nrUE --ninja --build-lib all -w USRP --verbose-ci

docker/Dockerfile.build.ubuntu18

@@ -35,4 +35,4 @@ COPY . .
 RUN /bin/sh oaienv && \
     cd cmake_targets && \
     mkdir -p log && \
-    ./build_oai --eNB --gNB --RU --UE --nrUE --ninja -w USRP --verbose-ci
+    ./build_oai --eNB --gNB --RU --UE --nrUE --ninja --build-lib all -w USRP --verbose-ci

docker/Dockerfile.eNB.rhel8.2

@@ -69,6 +69,8 @@ COPY --from=enb-build \
     /oai-ran/cmake_targets/ran_build/build/libparams_libconfig.so \
     /oai-ran/cmake_targets/ran_build/build/libdfts.so \
     /oai-ran/cmake_targets/ran_build/build/liboai_iqplayer.so \
+    /oai-ran/cmake_targets/ran_build/build/libtelnetsrv.so \
+    /oai-ran/cmake_targets/ran_build/build/libtelnetsrv_enb.so \
     /usr/local/lib/
 
 COPY --from=enb-base \

docker/Dockerfile.eNB.ubuntu18

@@ -76,6 +76,8 @@ COPY --from=enb-build \
      /oai-ran/cmake_targets/ran_build/build/libparams_libconfig.so \
      /oai-ran/cmake_targets/ran_build/build/libdfts.so \
      /oai-ran/cmake_targets/ran_build/build/liboai_iqplayer.so \
+     /oai-ran/cmake_targets/ran_build/build/libtelnetsrv.so \
+     /oai-ran/cmake_targets/ran_build/build/libtelnetsrv_enb.so \
      /usr/local/lib/
 
 # Now we are copying from builder-image the UHD files.

docker/Dockerfile.gNB.rhel8.2

@@ -74,6 +74,7 @@ COPY --from=gnb-build \
     /oai-ran/cmake_targets/ran_build/build/libldpc_optim.so \
     /oai-ran/cmake_targets/ran_build/build/libldpc_optim8seg.so \
     /oai-ran/cmake_targets/ran_build/build/libldpc_orig.so \
+    /oai-ran/cmake_targets/ran_build/build/libtelnetsrv.so \
     /usr/local/lib/
 
 COPY --from=gnb-base \

docker/Dockerfile.gNB.ubuntu18

@@ -78,6 +78,7 @@ COPY --from=gnb-build \
     /oai-ran/cmake_targets/ran_build/build/libldpc_optim.so \
     /oai-ran/cmake_targets/ran_build/build/libldpc_optim8seg.so \
     /oai-ran/cmake_targets/ran_build/build/libldpc_orig.so \
+    /oai-ran/cmake_targets/ran_build/build/libtelnetsrv.so \
     /usr/local/lib/
 
 # Now we are copying from builder-image the UHD files.

docker/Dockerfile.lteUE.rhel8.2

@@ -73,6 +73,7 @@ COPY --from=lte-ue-build \
     /oai-ran/cmake_targets/ran_build/build/libparams_libconfig.so \
     /oai-ran/cmake_targets/ran_build/build/libSIMU.so \
     /oai-ran/cmake_targets/ran_build/build/libdfts.so \
+    /oai-ran/cmake_targets/ran_build/build/libtelnetsrv.so \
     /usr/local/lib/
 
 COPY --from=lte-ue-base \

docker/Dockerfile.lteUE.ubuntu18

@@ -80,6 +80,7 @@ COPY --from=lte-ue-build \
     /oai-ran/cmake_targets/ran_build/build/libparams_libconfig.so \
     /oai-ran/cmake_targets/ran_build/build/libSIMU.so \
     /oai-ran/cmake_targets/ran_build/build/libdfts.so \
+    /oai-ran/cmake_targets/ran_build/build/libtelnetsrv.so \
     /usr/local/lib/
 
 # Now we are copying from builder-image the UHD files.

docker/Dockerfile.nrUE.rhel8.2

@@ -76,6 +76,8 @@ COPY --from=nr-ue-build \
     /oai-ran/cmake_targets/ran_build/build/libldpc_optim.so \
     /oai-ran/cmake_targets/ran_build/build/libldpc_optim8seg.so \
     /oai-ran/cmake_targets/ran_build/build/libldpc_orig.so \
+    /oai-ran/cmake_targets/ran_build/build/libtelnetsrv.so \
+    /oai-ran/cmake_targets/ran_build/build/libtelnetsrv_5Gue.so \
     /usr/local/lib/
 
 COPY --from=nr-ue-base \

docker/Dockerfile.nrUE.ubuntu18

@@ -81,6 +81,8 @@ COPY --from=nr-ue-build \
     /oai-ran/cmake_targets/ran_build/build/libldpc_optim.so \
     /oai-ran/cmake_targets/ran_build/build/libldpc_optim8seg.so \
     /oai-ran/cmake_targets/ran_build/build/libldpc_orig.so \
+    /oai-ran/cmake_targets/ran_build/build/libtelnetsrv.so \
+    /oai-ran/cmake_targets/ran_build/build/libtelnetsrv_5Gue.so \
     /usr/local/lib/
 
 # Now we are copying from builder-image the UHD files.