ran.py 59.4 KB
Newer Older
Raphael Defosseux's avatar
Raphael Defosseux committed
1
#/*
Gabriele Perrone's avatar
Gabriele Perrone committed
2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30
# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
# * contributor license agreements.  See the NOTICE file distributed with
# * this work for additional information regarding copyright ownership.
# * The OpenAirInterface Software Alliance licenses this file to You under
# * the OAI Public License, Version 1.1  (the "License"); you may not use this file
# * except in compliance with the License.
# * You may obtain a copy of the License at
# *
# *      http://www.openairinterface.org/?page_id=698
# *
# * Unless required by applicable law or agreed to in writing, software
# * distributed under the License is distributed on an "AS IS" BASIS,
# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# * See the License for the specific language governing permissions and
# * limitations under the License.
# *-------------------------------------------------------------------------------
# * For more information about the OpenAirInterface (OAI) Software Alliance:
# *      contact@openairinterface.org
# */
#---------------------------------------------------------------------
# Python for CI of OAI-eNB + COTS-UE
#
#   Required Python Version
#     Python 3.x
#
#   Required Python Package
#     pexpect
#---------------------------------------------------------------------

31 32 33 34 35 36 37 38 39
#-----------------------------------------------------------
# Import
#-----------------------------------------------------------
import sys              # arg
import re               # reg
import logging
import os
import time
from multiprocessing import Process, Lock, SimpleQueue
hardy's avatar
hardy committed
40
import yaml
41 42 43 44

#-----------------------------------------------------------
# OAI Testing modules
#-----------------------------------------------------------
Gabriele Perrone's avatar
Gabriele Perrone committed
45
import sshconnection as SSH
46 47 48 49 50 51 52
import helpreadme as HELP
import constants as CONST

#-----------------------------------------------------------
# Class Declaration
#-----------------------------------------------------------
class RANManagement():
Gabriele Perrone's avatar
Gabriele Perrone committed
53

54
	def __init__(self):
Gabriele Perrone's avatar
Gabriele Perrone committed
55 56
		
		self.prematureExit = False
57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78
		self.ranRepository = ''
		self.ranBranch = ''
		self.ranAllowMerge = False
		self.ranCommitID = ''
		self.ranTargetBranch = ''
		self.eNBIPAddress = ''
		self.eNBUserName = ''
		self.eNBPassword = ''
		self.eNBSourceCodePath = ''
		self.eNB1IPAddress = ''
		self.eNB1UserName = ''
		self.eNB1Password = ''
		self.eNB1SourceCodePath = ''
		self.eNB2IPAddress = ''
		self.eNB2UserName = ''
		self.eNB2Password = ''
		self.eNB2SourceCodePath = ''
		self.Build_eNB_args = ''
		self.backgroundBuild = False
		self.backgroundBuildTestId = ['', '', '']
		self.Build_eNB_forced_workspace_cleanup = False
		self.Initialize_eNB_args = ''
79
		self.imageKind = ''
80 81
		self.air_interface = ['', '', ''] #changed from 'lte' to '' may lead to side effects in main
		self.eNB_instance = 0
82
		self.eNB_serverId = ['', '', '']
83 84 85
		self.eNBLogFiles = ['', '', '']
		self.eNBOptions = ['', '', '']
		self.eNBmbmsEnables = [False, False, False]
86 87 88
		self.eNBstatuses = [-1, -1, -1]
		self.flexranCtrlInstalled = False
		self.flexranCtrlStarted = False
89 90
		self.flexranCtrlDeployed = False
		self.flexranCtrlIpAddress = ''
91
		self.testCase_id = ''
92
		self.epcPcapFile = ''
93
		self.runtime_stats= ''
hardy's avatar
hardy committed
94
		self.datalog_rt_stats={}
95
		self.eNB_Trace = '' #if 'yes', Tshark will be launched at initialization
hardy's avatar
hardy committed
96
		self.eNB_Stats = '' #if 'yes', Statistics Monitor will be launched at initialization		
hardy's avatar
hardy committed
97
		self.USRPIPAddress = ''
Gabriele Perrone's avatar
Gabriele Perrone committed
98

hardy's avatar
hardy committed
99

100

Gabriele Perrone's avatar
Gabriele Perrone committed
101 102 103 104
#-----------------------------------------------------------
# RAN management functions
#-----------------------------------------------------------

Raphael Defosseux's avatar
Raphael Defosseux committed
105
	def BuildeNB(self, HTML):
106
		if self.ranRepository == '' or self.ranBranch == '' or self.ranCommitID == '':
107
			HELP.GenericHelp(CONST.Version)
108
			sys.exit('Insufficient Parameter')
109
		if self.eNB_serverId[self.eNB_instance] == '0':
110 111 112 113
			lIpAddr = self.eNBIPAddress
			lUserName = self.eNBUserName
			lPassWord = self.eNBPassword
			lSourcePath = self.eNBSourceCodePath
114
		elif self.eNB_serverId[self.eNB_instance] == '1':
115 116 117 118
			lIpAddr = self.eNB1IPAddress
			lUserName = self.eNB1UserName
			lPassWord = self.eNB1Password
			lSourcePath = self.eNB1SourceCodePath
119
		elif self.eNB_serverId[self.eNB_instance] == '2':
120 121 122 123 124
			lIpAddr = self.eNB2IPAddress
			lUserName = self.eNB2UserName
			lPassWord = self.eNB2Password
			lSourcePath = self.eNB2SourceCodePath
		if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '':
125
			HELP.GenericHelp(CONST.Version)
126
			sys.exit('Insufficient Parameter')
127
		logging.debug('Building on server: ' + lIpAddr)
128
		mySSH = SSH.SSHConnection()
Gabriele Perrone's avatar
Gabriele Perrone committed
129
		mySSH.open(lIpAddr, lUserName, lPassWord)
hardy's avatar
hardy committed
130 131 132
		
		# Check if we build an 5G-NR gNB or an LTE eNB or an OCP eNB
		result = re.search('--eNBocp', self.Build_eNB_args)
133
		if result is not None:
134
			self.air_interface[self.eNB_instance] = 'ocp-enb'
135 136
		else:
			result = re.search('--RU', self.Build_eNB_args)
hardy's avatar
hardy committed
137
			if result is not None:
138
				self.air_interface[self.eNB_instance] = 'oairu'
hardy's avatar
hardy committed
139
			else:
140 141 142 143 144
				result = re.search('--gNB', self.Build_eNB_args)
				if result is not None:
					self.air_interface[self.eNB_instance] = 'nr-softmodem'
				else:
					self.air_interface[self.eNB_instance] = 'lte-softmodem'
145
		
146 147 148
		# Worakround for some servers, we need to erase completely the workspace
		if self.Build_eNB_forced_workspace_cleanup:
			mySSH.command('echo ' + lPassWord + ' | sudo -S rm -Rf ' + lSourcePath, '\$', 15)
Raphael Defosseux's avatar
Raphael Defosseux committed
149
		self.testCase_id = HTML.testCase_id
150 151 152
		# on RedHat/CentOS .git extension is mandatory
		result = re.search('([a-zA-Z0-9\:\-\.\/])+\.git', self.ranRepository)
		if result is not None:
153
			full_ran_repo_name = self.ranRepository.replace('git/', 'git')
154 155 156 157 158 159 160 161 162 163 164 165 166 167
		else:
			full_ran_repo_name = self.ranRepository + '.git'
		mySSH.command('mkdir -p ' + lSourcePath, '\$', 5)
		mySSH.command('cd ' + lSourcePath, '\$', 5)
		mySSH.command('if [ ! -e .git ]; then stdbuf -o0 git clone ' + full_ran_repo_name + ' .; else stdbuf -o0 git fetch --prune; fi', '\$', 600)
		# Raphael: here add a check if git clone or git fetch went smoothly
		mySSH.command('git config user.email "jenkins@openairinterface.org"', '\$', 5)
		mySSH.command('git config user.name "OAI Jenkins"', '\$', 5)
		# Checking the BUILD INFO file
		if not self.backgroundBuild:
			mySSH.command('ls *.txt', '\$', 5)
			result = re.search('LAST_BUILD_INFO', mySSH.getBefore())
			if result is not None:
				mismatch = False
168
				mySSH.command('grep --colour=never SRC_COMMIT LAST_BUILD_INFO.txt', '\$', 2)
169 170 171
				result = re.search(self.ranCommitID, mySSH.getBefore())
				if result is None:
					mismatch = True
172
				mySSH.command('grep --colour=never MERGED_W_TGT_BRANCH LAST_BUILD_INFO.txt', '\$', 2)
173 174 175 176
				if (self.ranAllowMerge):
					result = re.search('YES', mySSH.getBefore())
					if result is None:
						mismatch = True
177
					mySSH.command('grep --colour=never TGT_BRANCH LAST_BUILD_INFO.txt', '\$', 2)
178 179 180 181 182 183 184 185 186 187 188 189
					if self.ranTargetBranch == '':
						result = re.search('develop', mySSH.getBefore())
					else:
						result = re.search(self.ranTargetBranch, mySSH.getBefore())
					if result is None:
						mismatch = True
				else:
					result = re.search('NO', mySSH.getBefore())
					if result is None:
						mismatch = True
				if not mismatch:
					mySSH.close()
Raphael Defosseux's avatar
Raphael Defosseux committed
190
					HTML.CreateHtmlTestRow(self.Build_eNB_args, 'OK', CONST.ALL_PROCESSES_OK)
191 192 193 194 195
					return

		mySSH.command('echo ' + lPassWord + ' | sudo -S git clean -x -d -ff', '\$', 30)
		# if the commit ID is provided use it to point to it
		if self.ranCommitID != '':
196
			mySSH.command('git checkout -f ' + self.ranCommitID, '\$', 30)
197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213
		# if the branch is not develop, then it is a merge request and we need to do 
		# the potential merge. Note that merge conflicts should already been checked earlier
		if (self.ranAllowMerge):
			if self.ranTargetBranch == '':
				if (self.ranBranch != 'develop') and (self.ranBranch != 'origin/develop'):
					mySSH.command('git merge --ff origin/develop -m "Temporary merge for CI"', '\$', 5)
			else:
				logging.debug('Merging with the target branch: ' + self.ranTargetBranch)
				mySSH.command('git merge --ff origin/' + self.ranTargetBranch + ' -m "Temporary merge for CI"', '\$', 5)
		mySSH.command('source oaienv', '\$', 5)
		mySSH.command('cd cmake_targets', '\$', 5)
		mySSH.command('mkdir -p log', '\$', 5)
		mySSH.command('chmod 777 log', '\$', 5)
		# no need to remove in log (git clean did the trick)
		if self.backgroundBuild:
			mySSH.command('echo "./build_oai ' + self.Build_eNB_args + '" > ./my-lte-softmodem-build.sh', '\$', 5)
			mySSH.command('chmod 775 ./my-lte-softmodem-build.sh', '\$', 5)
214
			mySSH.command('echo ' + lPassWord + ' | sudo -S ls', '\$', 5)
215
			mySSH.command('echo $USER; nohup sudo -E ./my-lte-softmodem-build.sh' + ' > ' + lSourcePath + '/cmake_targets/compile_oai_enb.log ' + ' 2>&1 &', lUserName, 5)
216
			mySSH.close()
Raphael Defosseux's avatar
Raphael Defosseux committed
217
			HTML.CreateHtmlTestRow(self.Build_eNB_args, 'OK', CONST.ALL_PROCESSES_OK)
218
			self.backgroundBuildTestId[int(self.eNB_instance)] = self.testCase_id
219 220
			return
		mySSH.command('stdbuf -o0 ./build_oai ' + self.Build_eNB_args + ' 2>&1 | stdbuf -o0 tee compile_oai_enb.log', 'Bypassing the Tests|build have failed', 1500)
221
		mySSH.close()
Raphael Defosseux's avatar
Raphael Defosseux committed
222
		self.checkBuildeNB(lIpAddr, lUserName, lPassWord, lSourcePath, self.testCase_id, HTML)
223

Raphael Defosseux's avatar
Raphael Defosseux committed
224
	def WaitBuildeNBisFinished(self, HTML):
225
		if self.eNB_serverId[self.eNB_instance] == '0':
226 227 228 229
			lIpAddr = self.eNBIPAddress
			lUserName = self.eNBUserName
			lPassWord = self.eNBPassword
			lSourcePath = self.eNBSourceCodePath
230
		elif self.eNB_serverId[self.eNB_instance] == '1':
231 232 233 234
			lIpAddr = self.eNB1IPAddress
			lUserName = self.eNB1UserName
			lPassWord = self.eNB1Password
			lSourcePath = self.eNB1SourceCodePath
235
		elif self.eNB_serverId[self.eNB_instance] == '2':
236 237 238 239 240
			lIpAddr = self.eNB2IPAddress
			lUserName = self.eNB2UserName
			lPassWord = self.eNB2Password
			lSourcePath = self.eNB2SourceCodePath
		if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '':
241
			HELP.GenericHelp(CONST.Version)
242
			sys.exit('Insufficient Parameter')
243
		logging.debug('Waiting for end of build on server: ' + lIpAddr)
Gabriele Perrone's avatar
Gabriele Perrone committed
244
		mySSH = SSH.SSHConnection()
245 246 247 248
		mySSH.open(lIpAddr, lUserName, lPassWord)
		count = 40
		buildOAIprocess = True
		while (count > 0) and buildOAIprocess:
249
			mySSH.command('ps aux | grep --color=never build_ | grep -v grep', '\$', 6)
250 251 252 253 254 255
			result = re.search('build_oai', mySSH.getBefore())
			if result is None:
				buildOAIprocess = False
			else:
				count -= 1
				time.sleep(30)
256
		mySSH.close()
Raphael Defosseux's avatar
Raphael Defosseux committed
257
		self.checkBuildeNB(lIpAddr, lUserName, lPassWord, lSourcePath, self.backgroundBuildTestId[int(self.eNB_instance)], HTML)
258

Raphael Defosseux's avatar
Raphael Defosseux committed
259 260
	def checkBuildeNB(self, lIpAddr, lUserName, lPassWord, lSourcePath, testcaseId, HTML):
		HTML.testCase_id=testcaseId
hardy's avatar
hardy committed
261

262
		mySSH = SSH.SSHConnection()
263
		mySSH.open(lIpAddr, lUserName, lPassWord)
Gabriele Perrone's avatar
Gabriele Perrone committed
264
		mySSH.command('cd ' + lSourcePath + '/cmake_targets', '\$', 3)
265 266
		mySSH.command('ls ran_build/build', '\$', 3)
		mySSH.command('ls ran_build/build', '\$', 3)
hardy's avatar
hardy committed
267 268

		#check if we have the build corresponding to the air interface keywords (nr-softmode, lte-softmodem, ocp-enb)
269
		logging.info('CHECK Build with IP='+lIpAddr+' SourcePath='+lSourcePath)
270
		result = re.search(self.air_interface[self.eNB_instance], mySSH.getBefore())
271
		if result is None:
hardy's avatar
hardy committed
272
			buildStatus = False #if not, build failed
273
		else:
hardy's avatar
hardy committed
274
			buildStatus = True 
275 276 277 278 279 280 281 282 283 284 285
			# Generating a BUILD INFO file
			mySSH.command('echo "SRC_BRANCH: ' + self.ranBranch + '" > ../LAST_BUILD_INFO.txt', '\$', 2)
			mySSH.command('echo "SRC_COMMIT: ' + self.ranCommitID + '" >> ../LAST_BUILD_INFO.txt', '\$', 2)
			if (self.ranAllowMerge):
				mySSH.command('echo "MERGED_W_TGT_BRANCH: YES" >> ../LAST_BUILD_INFO.txt', '\$', 2)
				if self.ranTargetBranch == '':
					mySSH.command('echo "TGT_BRANCH: develop" >> ../LAST_BUILD_INFO.txt', '\$', 2)
				else:
					mySSH.command('echo "TGT_BRANCH: ' + self.ranTargetBranch + '" >> ../LAST_BUILD_INFO.txt', '\$', 2)
			else:
				mySSH.command('echo "MERGED_W_TGT_BRANCH: NO" >> ../LAST_BUILD_INFO.txt', '\$', 2)
hardy's avatar
hardy committed
286 287
				
				
288 289 290
		mySSH.command('mkdir -p build_log_' + testcaseId, '\$', 5)
		mySSH.command('mv log/* ' + 'build_log_' + testcaseId, '\$', 5)
		mySSH.command('mv compile_oai_enb.log ' + 'build_log_' + testcaseId, '\$', 5)
291
		if self.eNB_serverId[self.eNB_instance] != '0':
292 293 294 295 296 297 298 299 300 301 302 303
			mySSH.command('cd cmake_targets', '\$', 5)
			mySSH.command('if [ -e tmp_build' + testcaseId + '.zip ]; then rm -f tmp_build' + testcaseId + '.zip; fi', '\$', 5)
			mySSH.command('zip -r -qq tmp_build' + testcaseId + '.zip build_log_' + testcaseId, '\$', 5)
			mySSH.close()
			if (os.path.isfile('./tmp_build' + testcaseId + '.zip')):
				os.remove('./tmp_build' + testcaseId + '.zip')
			mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/tmp_build' + testcaseId + '.zip', '.')
			if (os.path.isfile('./tmp_build' + testcaseId + '.zip')):
				mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, './tmp_build' + testcaseId + '.zip', self.eNBSourceCodePath + '/cmake_targets/.')
				os.remove('./tmp_build' + testcaseId + '.zip')
				mySSH.open(self.eNBIPAddress, self.eNBUserName, self.eNBPassword)
				mySSH.command('cd ' + self.eNBSourceCodePath + '/cmake_targets', '\$', 5)
Remi Hardy's avatar
Remi Hardy committed
304
				#-qq quiet / -u update orcreate files
305
				mySSH.command('unzip -o -u -qq -DD tmp_build' + testcaseId + '.zip', '\$', 5)
306 307 308 309 310
				mySSH.command('rm -f tmp_build' + testcaseId + '.zip', '\$', 5)
				mySSH.close()
		else:
			mySSH.close()

hardy's avatar
hardy committed
311
		#generate logging info depending on buildStatus and air interface
312
		if buildStatus:
313
			logging.info('\u001B[1m Building OAI ' + self.air_interface[self.eNB_instance] + ' Pass\u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
314
			HTML.CreateHtmlTestRow(self.Build_eNB_args, 'OK', CONST.ALL_PROCESSES_OK)
315
		else:
316
			logging.error('\u001B[1m Building OAI ' + self.air_interface[self.eNB_instance] + ' Failed\u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
317 318
			HTML.CreateHtmlTestRow(self.Build_eNB_args, 'KO', CONST.ALL_PROCESSES_OK)
			HTML.CreateHtmlTabFooter(False)
319 320
			sys.exit(1)

Raphael Defosseux's avatar
Raphael Defosseux committed
321
	def InitializeeNB(self, HTML, EPC):
322
		if self.eNB_serverId[self.eNB_instance] == '0':
323 324 325 326
			lIpAddr = self.eNBIPAddress
			lUserName = self.eNBUserName
			lPassWord = self.eNBPassword
			lSourcePath = self.eNBSourceCodePath
327
		elif self.eNB_serverId[self.eNB_instance] == '1':
328 329 330 331
			lIpAddr = self.eNB1IPAddress
			lUserName = self.eNB1UserName
			lPassWord = self.eNB1Password
			lSourcePath = self.eNB1SourceCodePath
332
		elif self.eNB_serverId[self.eNB_instance] == '2':
333 334 335 336 337
			lIpAddr = self.eNB2IPAddress
			lUserName = self.eNB2UserName
			lPassWord = self.eNB2Password
			lSourcePath = self.eNB2SourceCodePath
		if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '':
338
			HELP.GenericHelp(CONST.Version)
339
			sys.exit('Insufficient Parameter')
340
		logging.debug('Starting eNB/gNB on server: ' + lIpAddr)
341

Raphael Defosseux's avatar
Raphael Defosseux committed
342
		self.testCase_id = HTML.testCase_id
Gabriele Perrone's avatar
Gabriele Perrone committed
343
		mySSH = SSH.SSHConnection()
hardy's avatar
hardy committed
344
		cwd = os.getcwd()
hardy's avatar
hardy committed
345
		mySSH.copyout(lIpAddr,lUserName,lPassWord, cwd + "/active_net_interfaces.awk", "/tmp")
Gabriele Perrone's avatar
Gabriele Perrone committed
346
		
hardy's avatar
hardy committed
347 348 349 350 351 352 353 354 355 356 357
		#reboot USRP if requested in xml
		if self.USRPIPAddress!='':
			logging.debug('USRP '+ self.USRPIPAddress +'reboot request')
			mySSH.open(lIpAddr, lUserName, lPassWord)
			cmd2usrp='ssh root@'+self.USRPIPAddress+' reboot'
			mySSH.command2(cmd2usrp,1)
			mySSH.close()
			logging.debug('Waiting for USRP to be ready')
			time.sleep(120)


Gabriele Perrone's avatar
Gabriele Perrone committed
358
		if (self.pStatus < 0):
Raphael Defosseux's avatar
Raphael Defosseux committed
359 360
			HTML.CreateHtmlTestRow(self.air_interface[self.eNB_instance] + ' ' + self.Initialize_eNB_args, 'KO', self.pStatus)
			HTML.CreateHtmlTabFooter(False)
361
			sys.exit(1)
hardy's avatar
hardy committed
362 363 364 365 366 367 368

		#Get pcap on enb and/or gnb if enabled in the xml 
		if self.eNB_Trace=='yes':
			if ((self.air_interface[self.eNB_instance] == 'lte-softmodem') or (self.air_interface[self.eNB_instance] == 'ocp-enb')):
				pcapfile_prefix="enb_"
			else:
				pcapfile_prefix="gnb_"
369 370 371 372 373 374
			mySSH.open(lIpAddr, lUserName, lPassWord)
			mySSH.command('ip addr show | awk -f /tmp/active_net_interfaces.awk | egrep -v "lo|tun"', '\$', 5)
			result = re.search('interfaceToUse=(?P<eth_interface>[a-zA-Z0-9\-\_]+)done', mySSH.getBefore())
			if result is not None:
				eth_interface = result.group('eth_interface')
				logging.debug('\u001B[1m Launching tshark on interface ' + eth_interface + '\u001B[0m')
hardy's avatar
hardy committed
375
				pcapfile = pcapfile_prefix + self.testCase_id + '_log.pcap'
376
				mySSH.command('echo ' + lPassWord + ' | sudo -S rm -f /tmp/' + pcapfile , '\$', 5)
377
				mySSH.command('echo $USER; nohup sudo -E tshark  -i ' + eth_interface + ' -w /tmp/' + pcapfile + ' > /dev/null 2>&1 &','\$', 5)
378 379 380 381
			mySSH.close()
			


382 383
		# If tracer options is on, running tshark on EPC side and capture traffic b/ EPC and eNB
		result = re.search('T_stdout', str(self.Initialize_eNB_args))
Raphael Defosseux's avatar
Raphael Defosseux committed
384 385 386 387
		if (result is not None):
			localEpcIpAddr = EPC.IPAddress
			localEpcUserName = EPC.UserName
			localEpcPassword = EPC.Password
388
			mySSH.open(localEpcIpAddr, localEpcUserName, localEpcPassword)
389 390 391 392 393
			mySSH.command('ip addr show | awk -f /tmp/active_net_interfaces.awk | egrep -v "lo|tun"', '\$', 5)
			result = re.search('interfaceToUse=(?P<eth_interface>[a-zA-Z0-9\-\_]+)done', mySSH.getBefore())
			if result is not None:
				eth_interface = result.group('eth_interface')
				logging.debug('\u001B[1m Launching tshark on interface ' + eth_interface + '\u001B[0m')
394 395 396
				self.epcPcapFile = 'enb_' + self.testCase_id + '_s1log.pcap'
				mySSH.command('echo ' + localEpcPassword + ' | sudo -S rm -f /tmp/' + self.epcPcapFile , '\$', 5)
				mySSH.command('echo $USER; nohup sudo tshark -f "host ' + lIpAddr +'" -i ' + eth_interface + ' -w /tmp/' + self.epcPcapFile + ' > /tmp/tshark.log 2>&1 &', localEpcUserName, 5)
397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425
			mySSH.close()
		mySSH.open(lIpAddr, lUserName, lPassWord)
		mySSH.command('cd ' + lSourcePath, '\$', 5)
		# Initialize_eNB_args usually start with -O and followed by the location in repository
		full_config_file = self.Initialize_eNB_args.replace('-O ','')
		extra_options = ''
		extIdx = full_config_file.find('.conf')
		if (extIdx > 0):
			extra_options = full_config_file[extIdx + 5:]
			# if tracer options is on, compiling and running T Tracer
			result = re.search('T_stdout', str(extra_options))
			if result is not None:
				logging.debug('\u001B[1m Compiling and launching T Tracer\u001B[0m')
				mySSH.command('cd common/utils/T/tracer', '\$', 5)
				mySSH.command('make', '\$', 10)
				mySSH.command('echo $USER; nohup ./record -d ../T_messages.txt -o ' + lSourcePath + '/cmake_targets/enb_' + self.testCase_id + '_record.raw -ON -off VCD -off HEAVY -off LEGACY_GROUP_TRACE -off LEGACY_GROUP_DEBUG > ' + lSourcePath + '/cmake_targets/enb_' + self.testCase_id + '_record.log 2>&1 &', lUserName, 5)
				mySSH.command('cd ' + lSourcePath, '\$', 5)
			full_config_file = full_config_file[:extIdx + 5]
			config_path, config_file = os.path.split(full_config_file)
		else:
			sys.exit('Insufficient Parameter')
		ci_full_config_file = config_path + '/ci-' + config_file
		rruCheck = False
		result = re.search('^rru|^rcc|^du.band', str(config_file))
		if result is not None:
			rruCheck = True
		# do not reset board twice in IF4.5 case
		result = re.search('^rru|^enb|^du.band', str(config_file))
		if result is not None:
426
			mySSH.command('echo ' + lPassWord + ' | sudo -S uhd_find_devices', '\$', 180)
427 428 429 430 431
			result = re.search('type: b200', mySSH.getBefore())
			if result is not None:
				logging.debug('Found a B2xx device --> resetting it')
				mySSH.command('echo ' + lPassWord + ' | sudo -S b2xx_fx3_utils --reset-device', '\$', 10)
				# Reloading FGPA bin firmware
432
				mySSH.command('echo ' + lPassWord + ' | sudo -S uhd_find_devices', '\$', 180)
433 434
		# Make a copy and adapt to EPC / eNB IP addresses
		mySSH.command('cp ' + full_config_file + ' ' + ci_full_config_file, '\$', 5)
Raphael Defosseux's avatar
Raphael Defosseux committed
435 436
		localMmeIpAddr = EPC.MmeIPAddress
		mySSH.command('sed -i -e \'s/CI_MME_IP_ADDR/' + localMmeIpAddr + '/\' ' + ci_full_config_file, '\$', 2);
437
		mySSH.command('sed -i -e \'s/CI_ENB_IP_ADDR/' + lIpAddr + '/\' ' + ci_full_config_file, '\$', 2);
438
		mySSH.command('sed -i -e \'s/CI_GNB_IP_ADDR/' + lIpAddr + '/\' ' + ci_full_config_file, '\$', 2);
439 440 441
		mySSH.command('sed -i -e \'s/CI_RCC_IP_ADDR/' + self.eNBIPAddress + '/\' ' + ci_full_config_file, '\$', 2);
		mySSH.command('sed -i -e \'s/CI_RRU1_IP_ADDR/' + self.eNB1IPAddress + '/\' ' + ci_full_config_file, '\$', 2);
		mySSH.command('sed -i -e \'s/CI_RRU2_IP_ADDR/' + self.eNB2IPAddress + '/\' ' + ci_full_config_file, '\$', 2);
442
		mySSH.command('sed -i -e \'s/CI_FR1_CTL_ENB_IP_ADDR/' + self.eNBIPAddress + '/\' ' + ci_full_config_file, '\$', 2);
443
		if (self.flexranCtrlInstalled and self.flexranCtrlStarted) or self.flexranCtrlDeployed:
444
			mySSH.command('sed -i -e \'s/FLEXRAN_ENABLED.*;/FLEXRAN_ENABLED        = "yes";/\' ' + ci_full_config_file, '\$', 2);
445
			mySSH.command('sed -i -e \'s/CI_FLEXRAN_CTL_IP_ADDR/' + self.flexranCtrlIpAddress + '/\' ' + ci_full_config_file, '\$', 2);
446 447 448
		else:
			mySSH.command('sed -i -e \'s/FLEXRAN_ENABLED.*;/FLEXRAN_ENABLED        = "no";/\' ' + ci_full_config_file, '\$', 2);
		self.eNBmbmsEnables[int(self.eNB_instance)] = False
449
		mySSH.command('grep --colour=never enable_enb_m2 ' + ci_full_config_file, '\$', 2);
450 451 452 453 454 455 456 457 458 459 460 461
		result = re.search('yes', mySSH.getBefore())
		if result is not None:
			self.eNBmbmsEnables[int(self.eNB_instance)] = True
			logging.debug('\u001B[1m MBMS is enabled on this eNB\u001B[0m')
		result = re.search('noS1', str(self.Initialize_eNB_args))
		eNBinNoS1 = False
		if result is not None:
			eNBinNoS1 = True
			logging.debug('\u001B[1m eNB is in noS1 configuration \u001B[0m')
		# Launch eNB with the modified config file
		mySSH.command('source oaienv', '\$', 5)
		mySSH.command('cd cmake_targets', '\$', 5)
462
		if self.air_interface[self.eNB_instance] == 'nr-softmodem':
463
			mySSH.command('if [ -e rbconfig.raw ]; then echo ' + lPassWord + ' | sudo -S rm rbconfig.raw; fi', '\$', 5)
464
			mySSH.command('if [ -e reconfig.raw ]; then echo ' + lPassWord + ' | sudo -S rm reconfig.raw; fi', '\$', 5)
465
		# NOTE: WE SHALL do a check if the executable is present (in case build went wrong)
466 467 468 469 470 471 472 473 474 475 476 477 478

		#hack UHD_RFNOC_DIR variable for gNB / N310 on RHEL8 server:
		#if the USRP address is in the xml then we are using an eth USRP (N3xx)
		if (self.air_interface[self.eNB_instance] == 'lte-softmodem') or (self.air_interface[self.eNB_instance] == 'ocp-enb'):
			gNB = False
		else:
			gNB = True
		if ((self.USRPIPAddress!='') and (gNB==True)):
			mySSH.command('echo ' + lPassWord + ' | echo "ulimit -c unlimited && sudo UHD_RFNOC_DIR=/usr/local/share/uhd/rfnoc ./ran_build/build/' + self.air_interface[self.eNB_instance] + ' -O ' + lSourcePath + '/' + ci_full_config_file + extra_options + '" > ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5)
		#otherwise the regular command is ok
		else:
			mySSH.command('echo "ulimit -c unlimited && ./ran_build/build/' + self.air_interface[self.eNB_instance] + ' -O ' + lSourcePath + '/' + ci_full_config_file + extra_options + '" > ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5)

479 480
		mySSH.command('chmod 775 ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5)
		mySSH.command('echo ' + lPassWord + ' | sudo -S rm -Rf enb_' + self.testCase_id + '.log', '\$', 5)
481
		mySSH.command('echo $USER; nohup sudo -E ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh > ' + lSourcePath + '/cmake_targets/enb_' + self.testCase_id + '.log 2>&1 &', lUserName, 10)
482 483


484
		#stats monitoring during runtime
485
		time.sleep(20)
486 487
		monitor_file='../ci-scripts/stats_monitor.py'
		conf_file='../ci-scripts/stats_monitor_conf.yaml'
488
		if self.eNB_Stats=='yes':
489
			if (self.air_interface[self.eNB_instance] == 'lte-softmodem') or (self.air_interface[self.eNB_instance] == 'ocp-enb'):
490
				mySSH.command('echo $USER; nohup python3 ' + monitor_file + ' ' + conf_file + ' enb 2>&1 > enb_stats_monitor_execution.log &', '\$', 5)
491
			else:
492
				mySSH.command('echo $USER; nohup python3 ' + monitor_file + ' ' + conf_file + ' gnb 2>&1 > gnb_stats_monitor_execution.log &', '\$', 5)
493 494 495



496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511
		self.eNBLogFiles[int(self.eNB_instance)] = 'enb_' + self.testCase_id + '.log'
		if extra_options != '':
			self.eNBOptions[int(self.eNB_instance)] = extra_options
		time.sleep(6)
		doLoop = True
		loopCounter = 20
		enbDidSync = False
		while (doLoop):
			loopCounter = loopCounter - 1
			if (loopCounter == 0):
				# In case of T tracer recording, we may need to kill it
				result = re.search('T_stdout', str(self.Initialize_eNB_args))
				if result is not None:
					mySSH.command('killall --signal SIGKILL record', '\$', 5)
				mySSH.close()
				doLoop = False
512
				logging.error('\u001B[1;37;41m eNB/gNB/ocp-eNB logging system did not show got sync! \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
513
				HTML.CreateHtmlTestRow(self.air_interface[self.eNB_instance] + ' -O ' + config_file + extra_options, 'KO', CONST.ALL_PROCESSES_OK)
514 515
				# In case of T tracer recording, we need to kill tshark on EPC side
				result = re.search('T_stdout', str(self.Initialize_eNB_args))
Raphael Defosseux's avatar
Raphael Defosseux committed
516 517 518 519
				if (result is not None):
					localEpcIpAddr = EPC.IPAddress
					localEpcUserName = EPC.UserName
					localEpcPassword = EPC.Password
520
					mySSH.open(localEpcIpAddr, localEpcUserName, localEpcPassword)
521
					logging.debug('\u001B[1m Stopping tshark \u001B[0m')
522 523
					mySSH.command('echo ' + localEpcPassword + ' | sudo -S killall --signal SIGKILL tshark', '\$', 5)
					if self.epcPcapFile  != '':
524
						time.sleep(0.5)
525
						mySSH.command('echo ' + localEpcPassword + ' | sudo -S chmod 666 /tmp/' + self.epcPcapFile, '\$', 5)
526 527
					mySSH.close()
					time.sleep(1)
528 529
					if self.epcPcapFile != '':
						copyin_res = mySSH.copyin(localEpcIpAddr, localEpcUserName, localEpcPassword, '/tmp/' + self.epcPcapFile, '.')
530
						if (copyin_res == 0):
531
							mySSH.copyout(lIpAddr, lUserName, lPassWord, self.epcPcapFile, lSourcePath + '/cmake_targets/.')
532 533 534
				self.prematureExit = True
				return
			else:
535
				mySSH.command('stdbuf -o0 cat enb_' + self.testCase_id + '.log | egrep --text --color=never -i "wait|sync|Starting|Started"', '\$', 4)
536 537 538 539 540 541 542 543 544 545 546
				if rruCheck:
					result = re.search('wait RUs', mySSH.getBefore())
				else:
					result = re.search('got sync|Starting F1AP at CU', mySSH.getBefore())
				if result is None:
					time.sleep(6)
				else:
					doLoop = False
					enbDidSync = True
					time.sleep(10)

547 548 549 550 551
		rruCheck = False
		result = re.search('^rru|^du.band', str(config_file))
		if result is not None:
			rruCheck = True
		if enbDidSync and eNBinNoS1 and not rruCheck:
552 553 554 555 556 557 558 559 560 561 562 563 564 565 566
			mySSH.command('ifconfig oaitun_enb1', '\$', 4)
			mySSH.command('ifconfig oaitun_enb1', '\$', 4)
			result = re.search('inet addr:1|inet 1', mySSH.getBefore())
			if result is not None:
				logging.debug('\u001B[1m oaitun_enb1 interface is mounted and configured\u001B[0m')
			else:
				logging.error('\u001B[1m oaitun_enb1 interface is either NOT mounted or NOT configured\u001B[0m')
			if self.eNBmbmsEnables[int(self.eNB_instance)]:
				mySSH.command('ifconfig oaitun_enm1', '\$', 4)
				result = re.search('inet addr', mySSH.getBefore())
				if result is not None:
					logging.debug('\u001B[1m oaitun_enm1 interface is mounted and configured\u001B[0m')
				else:
					logging.error('\u001B[1m oaitun_enm1 interface is either NOT mounted or NOT configured\u001B[0m')
		if enbDidSync:
567
			self.eNBstatuses[int(self.eNB_instance)] = int(self.eNB_serverId[self.eNB_instance])
568 569

		mySSH.close()
hardy's avatar
hardy committed
570 571


Raphael Defosseux's avatar
Raphael Defosseux committed
572
		HTML.CreateHtmlTestRow(self.air_interface[self.eNB_instance] + ' -O ' + config_file + extra_options, 'OK', CONST.ALL_PROCESSES_OK)
573
		logging.debug('\u001B[1m Initialize eNB/gNB/ocp-eNB Completed\u001B[0m')
574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593

	def CheckeNBProcess(self, status_queue):
		try:
			# At least the instance 0 SHALL be on!
			if self.eNBstatuses[0] == 0:
				lIpAddr = self.eNBIPAddress
				lUserName = self.eNBUserName
				lPassWord = self.eNBPassword
			elif self.eNBstatuses[0] == 1:
				lIpAddr = self.eNB1IPAddress
				lUserName = self.eNB1UserName
				lPassWord = self.eNB1Password
			elif self.eNBstatuses[0] == 2:
				lIpAddr = self.eNB2IPAddress
				lUserName = self.eNB2UserName
				lPassWord = self.eNB2Password
			else:
				lIpAddr = self.eNBIPAddress
				lUserName = self.eNBUserName
				lPassWord = self.eNBPassword
Gabriele Perrone's avatar
Gabriele Perrone committed
594
			mySSH = SSH.SSHConnection()
595
			mySSH.open(lIpAddr, lUserName, lPassWord)
596 597 598 599 600 601
			if self.air_interface[self.eNB_instance] == '':
				pattern = 'softmodem'
			else:
				pattern = self.air_interface[self.eNB_instance]
			mySSH.command('stdbuf -o0 ps -aux | grep --color=never ' + pattern + ' | grep -v grep', '\$', 5)
			result = re.search(pattern, mySSH.getBefore())
602 603
			if result is None:
				logging.debug('\u001B[1;37;41m eNB Process Not Found! \u001B[0m')
Gabriele Perrone's avatar
Gabriele Perrone committed
604
				status_queue.put(CONST.ENB_PROCESS_FAILED)
605
			else:
Gabriele Perrone's avatar
Gabriele Perrone committed
606
				status_queue.put(CONST.ENB_PROCESS_OK)
607 608 609 610
			mySSH.close()
		except:
			os.kill(os.getppid(),signal.SIGUSR1)

Raphael Defosseux's avatar
Raphael Defosseux committed
611
	def TerminateeNB(self, HTML, EPC):
612
		if self.eNB_serverId[self.eNB_instance] == '0':
613 614 615 616
			lIpAddr = self.eNBIPAddress
			lUserName = self.eNBUserName
			lPassWord = self.eNBPassword
			lSourcePath = self.eNBSourceCodePath
617
		elif self.eNB_serverId[self.eNB_instance] == '1':
618 619 620 621
			lIpAddr = self.eNB1IPAddress
			lUserName = self.eNB1UserName
			lPassWord = self.eNB1Password
			lSourcePath = self.eNB1SourceCodePath
622
		elif self.eNB_serverId[self.eNB_instance] == '2':
623 624 625 626 627
			lIpAddr = self.eNB2IPAddress
			lUserName = self.eNB2UserName
			lPassWord = self.eNB2Password
			lSourcePath = self.eNB2SourceCodePath
		if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '':
628 629
			HELP.GenericHelp(CONST.Version)
			sys.exit('Insufficient Parameter')
630
		logging.debug('Stopping eNB/gNB on server: ' + lIpAddr)
Gabriele Perrone's avatar
Gabriele Perrone committed
631
		mySSH = SSH.SSHConnection()
632 633
		mySSH.open(lIpAddr, lUserName, lPassWord)
		mySSH.command('cd ' + lSourcePath + '/cmake_targets', '\$', 5)
634
		if (self.air_interface[self.eNB_instance] == 'lte-softmodem') or (self.air_interface[self.eNB_instance] == 'ocp-enb'):
635 636 637
			nodeB_prefix = 'e'
		else:
			nodeB_prefix = 'g'
hardy's avatar
hardy committed
638 639
		mySSH.command('stdbuf -o0  ps -aux | grep --color=never -e softmodem -e ocp-enb | grep -v grep', '\$', 5)
		result = re.search('(-softmodem|ocp)', mySSH.getBefore())
640
		if result is not None:
Raphael Defosseux's avatar
Raphael Defosseux committed
641
			mySSH.command('echo ' + lPassWord + ' | sudo -S killall --signal SIGINT -r .*-softmodem ocp-enb || true', '\$', 5)
642
			time.sleep(10)
hardy's avatar
hardy committed
643 644
			mySSH.command('stdbuf -o0  ps -aux | grep --color=never -e softmodem -e ocp-enb | grep -v grep', '\$', 5)
			result = re.search('(-softmodem|ocp)', mySSH.getBefore())
645
			if result is not None:
Raphael Defosseux's avatar
Raphael Defosseux committed
646
				mySSH.command('echo ' + lPassWord + ' | sudo -S killall --signal SIGKILL -r .*-softmodem ocp-enb || true', '\$', 5)
647 648
				time.sleep(5)
		mySSH.command('rm -f my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5)
649
		#stopping tshark (valid if eNB and enabled in xml, will not harm otherwise)
650 651 652
		logging.debug('\u001B[1m Stopping tshark \u001B[0m')
		mySSH.command('echo ' + lPassWord + ' | sudo -S killall --signal SIGKILL tshark', '\$', 5)
		time.sleep(1)
653 654 655
		mySSH.close()
		# If tracer options is on, stopping tshark on EPC side
		result = re.search('T_stdout', str(self.Initialize_eNB_args))
Raphael Defosseux's avatar
Raphael Defosseux committed
656 657 658 659
		if (result is not None):
			localEpcIpAddr = EPC.IPAddress
			localEpcUserName = EPC.UserName
			localEpcPassword = EPC.Password
660
			mySSH.open(localEpcIpAddr, localEpcUserName, localEpcPassword)
661
			logging.debug('\u001B[1m Stopping tshark \u001B[0m')
662
			mySSH.command('echo ' + localEpcPassword + ' | sudo -S killall --signal SIGKILL tshark', '\$', 5)
663
			time.sleep(1)
664 665 666 667
			if self.epcPcapFile != '':
				mySSH.command('echo ' + localEpcPassword + ' | sudo -S chmod 666 /tmp/' + self.epcPcapFile, '\$', 5)
				mySSH.copyin(localEpcIpAddr, localEpcUserName, localEpcPassword, '/tmp/' + self.epcPcapFile, '.')
				mySSH.copyout(lIpAddr, lUserName, lPassWord, self.epcPcapFile, lSourcePath + '/cmake_targets/.')
668 669 670
			mySSH.close()
			logging.debug('\u001B[1m Replaying RAW record file\u001B[0m')
			mySSH.open(lIpAddr, lUserName, lPassWord)
671
			mySSH.command('killall --signal SIGKILL record', '\$', 5)
672 673 674 675 676 677 678 679 680 681 682 683
			mySSH.command('cd ' + lSourcePath + '/common/utils/T/tracer/', '\$', 5)
			enbLogFile = self.eNBLogFiles[int(self.eNB_instance)]
			raw_record_file = enbLogFile.replace('.log', '_record.raw')
			replay_log_file = enbLogFile.replace('.log', '_replay.log')
			extracted_txt_file = enbLogFile.replace('.log', '_extracted_messages.txt')
			extracted_log_file = enbLogFile.replace('.log', '_extracted_messages.log')
			mySSH.command('./extract_config -i ' + lSourcePath + '/cmake_targets/' + raw_record_file + ' > ' + lSourcePath + '/cmake_targets/' + extracted_txt_file, '\$', 5)
			mySSH.command('echo $USER; nohup ./replay -i ' + lSourcePath + '/cmake_targets/' + raw_record_file + ' > ' + lSourcePath + '/cmake_targets/' + replay_log_file + ' 2>&1 &', lUserName, 5)
			mySSH.command('./textlog -d ' +  lSourcePath + '/cmake_targets/' + extracted_txt_file + ' -no-gui -ON -full > ' + lSourcePath + '/cmake_targets/' + extracted_log_file, '\$', 5)
			mySSH.close()
			mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/' + extracted_log_file, '.')
			logging.debug('\u001B[1m Analyzing eNB replay logfile \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
684 685
			logStatus = self.AnalyzeLogFile_eNB(extracted_log_file, HTML)
			HTML.CreateHtmlTestRow(self.runtime_stats, 'OK', CONST.ALL_PROCESSES_OK)
686 687 688 689 690 691 692 693
			self.eNBLogFiles[int(self.eNB_instance)] = ''
		else:
			analyzeFile = False
			if self.eNBLogFiles[int(self.eNB_instance)] != '':
				analyzeFile = True
				fileToAnalyze = self.eNBLogFiles[int(self.eNB_instance)]
				self.eNBLogFiles[int(self.eNB_instance)] = ''
			if analyzeFile:
694 695 696 697 698
				#*stats.log files + pickle + png
				mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/*stats.log', '.')
				mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/*.pickle', '.')
				mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/*.png', '.')
				#
699 700 701
				copyin_res = mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/' + fileToAnalyze, '.')
				if (copyin_res == -1):
					logging.debug('\u001B[1;37;41m Could not copy ' + nodeB_prefix + 'NB logfile to analyze it! \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
702 703
					HTML.htmleNBFailureMsg='Could not copy ' + nodeB_prefix + 'NB logfile to analyze it!'
					HTML.CreateHtmlTestRow('N/A', 'KO', CONST.ENB_PROCESS_NOLOGFILE_TO_ANALYZE)
704 705
					self.eNBmbmsEnables[int(self.eNB_instance)] = False
					return
706
				if self.eNB_serverId[self.eNB_instance] != '0':
707
					#*stats.log files + pickle + png
hardy's avatar
hardy committed
708 709

					#debug / tentative
hardy's avatar
hardy committed
710 711
					mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, './nrL1_stats.log', self.eNBSourceCodePath + '/cmake_targets/')
					mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, './nrMAC_stats.log', self.eNBSourceCodePath + '/cmake_targets/')
hardy's avatar
hardy committed
712
					mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, './gnb_stats_monitor.pickle', self.eNBSourceCodePath + '/cmake_targets/')
hardy's avatar
hardy committed
713
					mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, './gnb_stats_monitor.png', self.eNBSourceCodePath + '/cmake_targets/')
714
					#
715 716
					mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, './' + fileToAnalyze, self.eNBSourceCodePath + '/cmake_targets/')
				logging.debug('\u001B[1m Analyzing ' + nodeB_prefix + 'NB logfile \u001B[0m ' + fileToAnalyze)
Raphael Defosseux's avatar
Raphael Defosseux committed
717
				logStatus = self.AnalyzeLogFile_eNB(fileToAnalyze, HTML)
718
				if (logStatus < 0):
Raphael Defosseux's avatar
Raphael Defosseux committed
719
					HTML.CreateHtmlTestRow('N/A', 'KO', logStatus)
hardy's avatar
hardy committed
720
					self.prematureExit = True
721 722 723
					self.eNBmbmsEnables[int(self.eNB_instance)] = False
					return
				else:
Raphael Defosseux's avatar
Raphael Defosseux committed
724
					HTML.CreateHtmlTestRow(self.runtime_stats, 'OK', CONST.ALL_PROCESSES_OK)
725
			else:
Raphael Defosseux's avatar
Raphael Defosseux committed
726
				HTML.CreateHtmlTestRow(self.runtime_stats, 'OK', CONST.ALL_PROCESSES_OK)
727 728
		#display rt stats for gNB only
		if len(self.datalog_rt_stats)!=0 and nodeB_prefix == 'g':
hardy's avatar
hardy committed
729
			HTML.CreateHtmlDataLogTable(self.datalog_rt_stats)
730 731 732 733
		self.eNBmbmsEnables[int(self.eNB_instance)] = False
		self.eNBstatuses[int(self.eNB_instance)] = -1

	def LogCollecteNB(self):
Gabriele Perrone's avatar
Gabriele Perrone committed
734
		mySSH = SSH.SSHConnection()
735 736 737
		mySSH.open(self.eNBIPAddress, self.eNBUserName, self.eNBPassword)
		mySSH.command('cd ' + self.eNBSourceCodePath, '\$', 5)
		mySSH.command('cd cmake_targets', '\$', 5)
hardy's avatar
hardy committed
738 739
		mySSH.command('echo ' + self.eNBPassword + ' | sudo -S mv /tmp/enb_*.pcap .','\$',20)
		mySSH.command('echo ' + self.eNBPassword + ' | sudo -S mv /tmp/gnb_*.pcap .','\$',20)
740
		mySSH.command('echo ' + self.eNBPassword + ' | sudo -S rm -f enb.log.zip', '\$', 5)
741 742
		mySSH.command('echo ' + self.eNBPassword + ' | sudo -S zip enb.log.zip enb*.log core* enb_*record.raw enb_*.pcap gnb_*.pcap enb_*txt physim_*.log *stats.log *monitor.pickle *monitor*.png log/*/*.log log/*/*.pcap', '\$', 60)
		mySSH.command('echo ' + self.eNBPassword + ' | sudo -S rm enb*.log core* enb_*record.raw enb_*.pcap gnb_*.pcap enb_*txt physim_*.log *stats.log *monitor.pickle *monitor*.png log/*/*.log log/*/*.pcap', '\$', 15)
743
		mySSH.close()
Gabriele Perrone's avatar
Gabriele Perrone committed
744

Raphael Defosseux's avatar
Raphael Defosseux committed
745
	def AnalyzeLogFile_eNB(self, eNBlogFile, HTML):
Gabriele Perrone's avatar
Gabriele Perrone committed
746 747 748 749 750 751 752 753 754
		if (not os.path.isfile('./' + eNBlogFile)):
			return -1
		enb_log_file = open('./' + eNBlogFile, 'r')
		exitSignalReceived = False
		foundAssertion = False
		msgAssertion = ''
		msgLine = 0
		foundSegFault = False
		foundRealTimeIssue = False
755
		foundRealTimeIssue_cnt = 0
Gabriele Perrone's avatar
Gabriele Perrone committed
756 757 758 759 760 761 762 763 764 765 766 767
		rrcSetupComplete = 0
		rrcReleaseRequest = 0
		rrcReconfigRequest = 0
		rrcReconfigComplete = 0
		rrcReestablishRequest = 0
		rrcReestablishComplete = 0
		rrcReestablishReject = 0
		rlcDiscardBuffer = 0
		rachCanceledProcedure = 0
		uciStatMsgCount = 0
		pdcpFailure = 0
		ulschFailure = 0
768 769
		ulschAllocateCCEerror = 0
		uplinkSegmentsAborted = 0
770
		ulschReceiveOK = 0
771
		gnbRxTxWakeUpFailure = 0
772
		gnbTxWriteThreadEnabled = False
Gabriele Perrone's avatar
Gabriele Perrone committed
773 774 775
		cdrxActivationMessageCount = 0
		dropNotEnoughRBs = 0
		mbmsRequestMsg = 0
776
		htmleNBFailureMsg = ''
Gabriele Perrone's avatar
Gabriele Perrone committed
777 778 779
		isRRU = False
		isSlave = False
		slaveReceivesFrameResyncCmd = False
Gabriele Perrone's avatar
Gabriele Perrone committed
780
		X2HO_state = CONST.X2_HO_REQ_STATE__IDLE
Gabriele Perrone's avatar
Gabriele Perrone committed
781 782
		X2HO_inNbProcedures = 0
		X2HO_outNbProcedures = 0
783
		global_status = CONST.ALL_PROCESSES_OK
784 785 786 787 788 789
		# Runtime statistics
		runTime = ''
		userTime = ''
		systemTime = ''
		maxPhyMemUsage = ''
		nbContextSwitches = ''
hardy's avatar
hardy committed
790
		#NSA FR1 check
hardy's avatar
hardy committed
791
		NSA_RAPROC_PUSCH_check = 0
hardy's avatar
hardy committed
792 793
		#dlsch and ulsch statistics (dictionary)
		dlsch_ulsch_stats = {}
794
		#real time statistics (dictionary)
795
		real_time_stats = {}
Remi Hardy's avatar
Remi Hardy committed
796 797
		#count "problem receiving samples" msg
		pb_receiving_samples_cnt = 0
798 799
		#count "removing UE" msg
		removing_ue = 0
hardy's avatar
hardy committed
800 801
		#count"X2AP-PDU"
		x2ap_pdu = 0
802 803
		#NSA specific log markers
		nsa_markers ={'SgNBReleaseRequestAcknowledge': [],'FAILURE': [], 'scgFailureInformationNR-r15': [], 'SgNBReleaseRequest': []}
804
		nodeB_prefix_found = False
805
	
806
		line_cnt=0 #log file line counter
Gabriele Perrone's avatar
Gabriele Perrone committed
807
		for line in enb_log_file.readlines():
808
			line_cnt+=1
809 810 811 812 813 814 815 816 817 818
			# Detection of eNB/gNB from a container log
			result = re.search('Starting eNB soft modem', str(line))
			if result is not None:
				nodeB_prefix_found = True
				nodeB_prefix = 'e'
			result = re.search('Starting gNB soft modem', str(line))
			if result is not None:
				nodeB_prefix_found = True
				nodeB_prefix = 'g'
			result = re.search('Run time:' ,str(line))
819 820 821 822 823 824 825
			# Runtime statistics
			result = re.search('Run time:' ,str(line))
			if result is not None:
				runTime = str(line).strip()
			if runTime != '':
				result = re.search('Time executing user inst', str(line))
				if result is not None:
826 827
					fields=line.split(':')
					userTime = 'userTime : ' + fields[1].replace('\n','')
828 829
				result = re.search('Time executing system inst', str(line))
				if result is not None:
830 831
					fields=line.split(':')
					systemTime = 'systemTime : ' + fields[1].replace('\n','')
832 833
				result = re.search('Max. Phy. memory usage:', str(line))
				if result is not None:
834 835
					fields=line.split(':')
					maxPhyMemUsage = 'maxPhyMemUsage : ' + fields[1].replace('\n','')
836 837
				result = re.search('Number of context switch.*process origin', str(line))
				if result is not None:
838 839
					fields=line.split(':')
					nbContextSwitches = 'nbContextSwitches : ' + fields[1].replace('\n','')
Gabriele Perrone's avatar
Gabriele Perrone committed
840
			if X2HO_state == CONST.X2_HO_REQ_STATE__IDLE:
Gabriele Perrone's avatar
Gabriele Perrone committed
841 842
				result = re.search('target eNB Receives X2 HO Req X2AP_HANDOVER_REQ', str(line))
				if result is not None:
Gabriele Perrone's avatar
Gabriele Perrone committed
843
					X2HO_state = CONST.X2_HO_REQ_STATE__TARGET_RECEIVES_REQ
Gabriele Perrone's avatar
Gabriele Perrone committed
844 845
				result = re.search('source eNB receives the X2 HO ACK X2AP_HANDOVER_REQ_ACK', str(line))
				if result is not None:
Gabriele Perrone's avatar
Gabriele Perrone committed
846 847
					X2HO_state = CONST.X2_HO_REQ_STATE__SOURCE_RECEIVES_REQ_ACK
			if X2HO_state == CONST.X2_HO_REQ_STATE__TARGET_RECEIVES_REQ:
Gabriele Perrone's avatar
Gabriele Perrone committed
848 849
				result = re.search('Received LTE_RRCConnectionReconfigurationComplete from UE', str(line))
				if result is not None:
Gabriele Perrone's avatar
Gabriele Perrone committed
850 851
					X2HO_state = CONST.X2_HO_REQ_STATE__TARGET_RRC_RECFG_COMPLETE
			if X2HO_state == CONST.X2_HO_REQ_STATE__TARGET_RRC_RECFG_COMPLETE:
Gabriele Perrone's avatar
Gabriele Perrone committed
852 853
				result = re.search('issue rrc_eNB_send_PATH_SWITCH_REQ', str(line))
				if result is not None:
Gabriele Perrone's avatar
Gabriele Perrone committed
854 855
					X2HO_state = CONST.X2_HO_REQ_STATE__TARGET_SENDS_SWITCH_REQ
			if X2HO_state == CONST.X2_HO_REQ_STATE__TARGET_SENDS_SWITCH_REQ:
Gabriele Perrone's avatar
Gabriele Perrone committed
856 857
				result = re.search('received path switch ack S1AP_PATH_SWITCH_REQ_ACK', str(line))
				if result is not None:
Gabriele Perrone's avatar
Gabriele Perrone committed
858
					X2HO_state = CONST.X2_HO_REQ_STATE__IDLE
Gabriele Perrone's avatar
Gabriele Perrone committed
859
					X2HO_inNbProcedures += 1
Gabriele Perrone's avatar
Gabriele Perrone committed
860
			if X2HO_state == CONST.X2_HO_REQ_STATE__SOURCE_RECEIVES_REQ_ACK:
Gabriele Perrone's avatar
Gabriele Perrone committed
861 862
				result = re.search('source eNB receives the X2 UE CONTEXT RELEASE X2AP_UE_CONTEXT_RELEASE', str(line))
				if result is not None:
Gabriele Perrone's avatar
Gabriele Perrone committed
863
					X2HO_state = CONST.X2_HO_REQ_STATE__IDLE
Gabriele Perrone's avatar
Gabriele Perrone committed
864 865 866 867 868 869 870 871 872
					X2HO_outNbProcedures += 1

			if self.eNBOptions[int(self.eNB_instance)] != '':
				res1 = re.search('max_rxgain (?P<requested_option>[0-9]+)', self.eNBOptions[int(self.eNB_instance)])
				res2 = re.search('max_rxgain (?P<applied_option>[0-9]+)',  str(line))
				if res1 is not None and res2 is not None:
					requested_option = int(res1.group('requested_option'))
					applied_option = int(res2.group('applied_option'))
					if requested_option == applied_option:
873
						htmleNBFailureMsg += '<span class="glyphicon glyphicon-ok-circle"></span> Command line option(s) correctly applied <span class="glyphicon glyphicon-arrow-right"></span> ' + self.eNBOptions[int(self.eNB_instance)] + '\n\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
874
					else:
875
						htmleNBFailureMsg += '<span class="glyphicon glyphicon-ban-circle"></span> Command line option(s) NOT applied <span class="glyphicon glyphicon-arrow-right"></span> ' + self.eNBOptions[int(self.eNB_instance)] + '\n\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893
			result = re.search('Exiting OAI softmodem', str(line))
			if result is not None:
				exitSignalReceived = True
			result = re.search('[Ss]egmentation [Ff]ault', str(line))
			if result is not None and not exitSignalReceived:
				foundSegFault = True
			result = re.search('[Cc]ore [dD]ump', str(line))
			if result is not None and not exitSignalReceived:
				foundSegFault = True
			result = re.search('./ran_build/build/lte-softmodem', str(line))
			if result is not None and not exitSignalReceived:
				foundSegFault = True
			result = re.search('[Aa]ssertion', str(line))
			if result is not None and not exitSignalReceived:
				foundAssertion = True
			result = re.search('LLL', str(line))
			if result is not None and not exitSignalReceived:
				foundRealTimeIssue = True
894
				foundRealTimeIssue_cnt += 1
Gabriele Perrone's avatar
Gabriele Perrone committed
895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912
			if foundAssertion and (msgLine < 3):
				msgLine += 1
				msgAssertion += str(line)
			result = re.search('Setting function for RU', str(line))
			if result is not None:
				isRRU = True
			if isRRU:
				result = re.search('RU 0 is_slave=yes', str(line))
				if result is not None:
					isSlave = True
				if isSlave:
					result = re.search('Received RRU_frame_resynch command', str(line))
					if result is not None:
						slaveReceivesFrameResyncCmd = True
			result = re.search('LTE_RRCConnectionSetupComplete from UE', str(line))
			if result is not None:
				rrcSetupComplete += 1
			result = re.search('Generate LTE_RRCConnectionRelease|Generate RRCConnectionRelease', str(line))
hardy's avatar
hardy committed
913
			if result is not None:				rrcReleaseRequest += 1
Gabriele Perrone's avatar
Gabriele Perrone committed
914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937
			result = re.search('Generate LTE_RRCConnectionReconfiguration', str(line))
			if result is not None:
				rrcReconfigRequest += 1
			result = re.search('LTE_RRCConnectionReconfigurationComplete from UE rnti', str(line))
			if result is not None:
				rrcReconfigComplete += 1
			result = re.search('LTE_RRCConnectionReestablishmentRequest', str(line))
			if result is not None:
				rrcReestablishRequest += 1
			result = re.search('LTE_RRCConnectionReestablishmentComplete', str(line))
			if result is not None:
				rrcReestablishComplete += 1
			result = re.search('LTE_RRCConnectionReestablishmentReject', str(line))
			if result is not None:
				rrcReestablishReject += 1
			result = re.search('CDRX configuration activated after RRC Connection', str(line))
			if result is not None:
				cdrxActivationMessageCount += 1
			result = re.search('uci->stat', str(line))
			if result is not None:
				uciStatMsgCount += 1
			result = re.search('PDCP.*Out of Resources.*reason', str(line))
			if result is not None:
				pdcpFailure += 1
938 939 940
			result = re.search('could not wakeup gNB rxtx process', str(line))
			if result is not None:
				gnbRxTxWakeUpFailure += 1
941 942 943
			result = re.search('tx write thread ready', str(line))
			if result is not None:
				gnbTxWriteThreadEnabled = True
944
			result = re.search('ULSCH in error in round|ULSCH 0 in error', str(line))
Gabriele Perrone's avatar
Gabriele Perrone committed
945 946
			if result is not None:
				ulschFailure += 1
947 948 949 950 951 952
			result = re.search('ERROR ALLOCATING CCEs', str(line))
			if result is not None:
				ulschAllocateCCEerror += 1
			result = re.search('uplink segment error.*aborted [1-9] segments', str(line))
			if result is not None:
				uplinkSegmentsAborted += 1
953 954 955
			result = re.search('ULSCH received ok', str(line))
			if result is not None:
				ulschReceiveOK += 1
Gabriele Perrone's avatar
Gabriele Perrone committed
956 957 958 959 960 961 962 963 964 965 966 967 968
			result = re.search('BAD all_segments_received', str(line))
			if result is not None:
				rlcDiscardBuffer += 1
			result = re.search('Canceled RA procedure for UE rnti', str(line))
			if result is not None:
				rachCanceledProcedure += 1
			result = re.search('dropping, not enough RBs', str(line))
			if result is not None:
				dropNotEnoughRBs += 1
			if self.eNBmbmsEnables[int(self.eNB_instance)]:
				result = re.search('MBMS USER-PLANE.*Requesting.*bytes from RLC', str(line))
				if result is not None:
					mbmsRequestMsg += 1
hardy's avatar
hardy committed
969
			#FR1 NSA test : add new markers to make sure gNB is used
970
			result = re.search('\[gNB [0-9]+\]\[RAPROC\] PUSCH with TC_RNTI 0x[0-9a-fA-F]+ received correctly, adding UE MAC Context UE_id [0-9]+\/RNTI 0x[0-9a-fA-F]+', str(line))
hardy's avatar
hardy committed
971 972
			if result is not None:
				NSA_RAPROC_PUSCH_check = 1
hardy's avatar
hardy committed
973
			#dlsch and ulsch statistics
974
			#keys below are the markers we are loooking for, loop over this keys list
hardy's avatar
hardy committed
975
			#everytime these markers are found in the log file, the previous ones are overwritten in the dict
Remi Hardy's avatar
Remi Hardy committed
976
			#eventually we record and print only the last occurence 
hardy's avatar
hardy committed
977
			keys = {'UE ID','dlsch_rounds','dlsch_total_bytes','ulsch_rounds','ulsch_total_bytes_scheduled'}
hardy's avatar
hardy committed
978
			for k in keys:
979
				result = re.search(k, line)
hardy's avatar
hardy committed
980
				if result is not None:
Remi Hardy's avatar
Remi Hardy committed
981 982
					#remove 1- all useless char before relevant info (ulsch or dlsch) 2- trailing char
					dlsch_ulsch_stats[k]=re.sub(r'^.*\]\s+', r'' , line.rstrip())
983

hardy's avatar
hardy committed
984

Remi Hardy's avatar
Remi Hardy committed
985 986 987
			#count "problem receiving samples" msg
			result = re.search('\[PHY\]\s+problem receiving samples', str(line))
			if result is not None:
988
				pb_receiving_samples_cnt += 1
989 990 991 992
			#count "Removing UE" msg
			result = re.search('\[MAC\]\s+Removing UE', str(line))
			if result is not None:
				removing_ue += 1
hardy's avatar
hardy committed
993 994 995 996
			#count "X2AP-PDU"
			result = re.search('X2AP-PDU', str(line))
			if result is not None:
				x2ap_pdu += 1
997 998 999 1000 1001
			#nsa markers logging
			for k in nsa_markers:
				result = re.search(k, line)
				if result is not None:
					nsa_markers[k].append(line_cnt)					
Remi Hardy's avatar
Remi Hardy committed
1002

Gabriele Perrone's avatar
Gabriele Perrone committed
1003
		enb_log_file.close()
1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024


		#the following part takes the *_stats.log files as source (not the stdout log file)

		#the datalog config file has to be loaded
		datalog_rt_stats_file='datalog_rt_stats.yaml'
		if (os.path.isfile(datalog_rt_stats_file)):
			yaml_file=datalog_rt_stats_file
		elif (os.path.isfile('ci-scripts/'+datalog_rt_stats_file)):
			yaml_file='ci-scripts/'+datalog_rt_stats_file
		else:
			logging.error("Datalog RT stats yaml file cannot be found")
			sys.exit("Datalog RT stats yaml file cannot be found")

		with open(yaml_file,'r') as f:
			datalog_rt_stats = yaml.load(f,Loader=yaml.FullLoader)
		rt_keys = datalog_rt_stats['Ref'] #we use the keys from the Ref field  

		if (os.path.isfile('./nrL1_stats.log')) and (os.path.isfile('./nrL1_stats.log')):
			stat_files_present=True
		else:
hardy's avatar
hardy committed
1025
			stat_files_present=False
1026 1027
			logging.debug("NR Stats files for RT analysis not found")
		if stat_files_present:
hardy's avatar
hardy committed
1028 1029 1030
			nrL1_stats = open('./nrL1_stats.log', 'r')
			nrMAC_stats = open('./nrMAC_stats.log', 'r')
			for line in nrL1_stats.readlines():
1031 1032 1033 1034 1035 1036 1037
				for k in rt_keys:
					result = re.search(k, line)     
					if result is not None:
						#remove 1- all useless char before relevant info  2- trailing char
						tmp=re.match(rf'^.*?(\b{k}\b.*)',line.rstrip()) #from python 3.6 we can use literal string interpolation for the variable k, using rf' in the regex
						if tmp!=None: 
							real_time_stats[k]=tmp.group(1)
hardy's avatar
hardy committed
1038
			for line in nrMAC_stats.readlines():
1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052
				for k in rt_keys:
					result = re.search(k, line)     
					if result is not None:
						#remove 1- all useless char before relevant info  2- trailing char
						tmp=re.match(rf'^.*?(\b{k}\b.*)',line.rstrip()) #from python 3.6 we can use literal string interpolation for the variable k, using rf' in the regex
						if tmp!=None: 
							real_time_stats[k]=tmp.group(1)
			nrL1_stats.close()
			nrMAC_stats.close()

		#stdout log file and stat log files analysis completed
		logging.debug('   File analysis (stdout, stats) completed')

		#post processing depending on the node type
1053 1054 1055 1056 1057
		if not nodeB_prefix_found:
			if (self.air_interface[self.eNB_instance] == 'lte-softmodem') or (self.air_interface[self.eNB_instance] == 'ocp-enb'):
				nodeB_prefix = 'e'
			else:
				nodeB_prefix = 'g'
hardy's avatar
hardy committed
1058

1059
		if nodeB_prefix == 'g':
1060 1061 1062 1063
			if ulschReceiveOK > 0:
				statMsg = nodeB_prefix + 'NB showed ' + str(ulschReceiveOK) + ' "ULSCH received ok" message(s)'
				logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
				htmleNBFailureMsg += statMsg + '\n'
1064 1065 1066 1067
			if gnbRxTxWakeUpFailure > 0:
				statMsg = nodeB_prefix + 'NB showed ' + str(gnbRxTxWakeUpFailure) + ' "could not wakeup gNB rxtx process" message(s)'
				logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
				htmleNBFailureMsg += statMsg + '\n'
1068 1069 1070
			if gnbTxWriteThreadEnabled:
				statMsg = nodeB_prefix + 'NB ran with TX Write thread enabled'
				logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
1071
				htmleNBFailureMsg += statMsg + '\n'
hardy's avatar
hardy committed
1072
			#FR1 NSA test : add new markers to make sure gNB is used
1073 1074 1075 1076
			if NSA_RAPROC_PUSCH_check:
				statMsg = '[RAPROC] PUSCH with TC_RNTI message check for ' + nodeB_prefix + 'NB : PASS '
				htmlMsg = statMsg+'\n'
			else:
1077
				statMsg = '[RAPROC] PUSCH with TC_RNTI message check for ' + nodeB_prefix + 'NB : FAIL or not relevant'
1078 1079 1080
				htmlMsg = statMsg+'\n'
			logging.debug(statMsg)
			htmleNBFailureMsg += htmlMsg
Remi Hardy's avatar
Remi Hardy committed
1081 1082 1083 1084 1085
			#problem receiving samples log
			statMsg = '[PHY] problem receiving samples msg count =  '+str(pb_receiving_samples_cnt)
			htmlMsg = statMsg+'\n'
			logging.debug(statMsg)
			htmleNBFailureMsg += htmlMsg
1086 1087 1088 1089 1090
			#nsa markers
			statMsg = 'logfile line count = ' + str(line_cnt)			
			htmlMsg = statMsg+'\n'
			logging.debug(statMsg)
			htmleNBFailureMsg += htmlMsg
1091 1092 1093 1094
			if len(nsa_markers['SgNBReleaseRequestAcknowledge'])!=0:
				statMsg = 'SgNBReleaseRequestAcknowledge = ' + str(len(nsa_markers['SgNBReleaseRequestAcknowledge'])) + ' occurences , starting line ' + str(nsa_markers['SgNBReleaseRequestAcknowledge'][0])
			else:
				statMsg = 'SgNBReleaseRequestAcknowledge = ' + str(len(nsa_markers['SgNBReleaseRequestAcknowledge'])) + ' occurences' 
1095 1096 1097 1098 1099 1100 1101
			htmlMsg = statMsg+'\n'
			logging.debug(statMsg)
			htmleNBFailureMsg += htmlMsg
			statMsg = 'FAILURE = ' + str(len(nsa_markers['FAILURE'])) + ' occurences'
			htmlMsg = statMsg+'\n'
			logging.debug(statMsg)
			htmleNBFailureMsg += htmlMsg
Remi Hardy's avatar
Remi Hardy committed
1102

1103 1104 1105 1106
			#ulsch and dlsch statistics
			if len(dlsch_ulsch_stats)!=0: #check if dictionary is not empty
				statMsg=''
				for key in dlsch_ulsch_stats: #for each dictionary key
Remi Hardy's avatar
Remi Hardy committed
1107
					statMsg += dlsch_ulsch_stats[key] + '\n' 
1108 1109
					logging.debug(dlsch_ulsch_stats[key])
				htmleNBFailureMsg += statMsg
hardy's avatar
hardy committed
1110

hardy's avatar
hardy committed
1111 1112
			#real time statistics
			datalog_rt_stats['Data']={}
1113
			if len(real_time_stats)!=0: #check if dictionary is not empty
hardy's avatar
hardy committed
1114 1115 1116 1117 1118 1119 1120
				for k in real_time_stats:
					tmp=re.match(r'^(?P<metric>.*):\s+(?P<avg>\d+\.\d+) us;\s+\d+;\s+(?P<max>\d+\.\d+) us;',real_time_stats[k])
					if tmp is not None:
						metric=tmp.group('metric')
						avg=float(tmp.group('avg'))
						max=float(tmp.group('max'))
						datalog_rt_stats['Data'][metric]=["{:.0f}".format(avg),"{:.0f}".format(max),"{:.2f}".format(avg/datalog_rt_stats['Ref'][metric])]
1121 1122 1123 1124 1125 1126
				#once all metrics are collected, store the data as a class attribute to build a dedicated HTML table afterward
				self.datalog_rt_stats=datalog_rt_stats
				#check if there is a fail => will render the test as failed
				for k in datalog_rt_stats['Data']:
					if float(datalog_rt_stats['Data'][k][2])> datalog_rt_stats['Threshold'][k]: #condition for fail : avg/ref is greater than the fixed threshold
						#setting prematureExit is ok although not the best option
1127
						self.prematureExit=False #temp for debug : do not stop the test if RT stats are excedeed
1128 1129 1130 1131
			else:
				statMsg = 'No real time stats found in the log file\n'
				logging.debug('No real time stats found in the log file')
				htmleNBFailureMsg += statMsg
1132

1133
		else:
1134 1135 1136 1137 1138
			#Removing UE log
			statMsg = '[MAC] Removing UE msg count =  '+str(removing_ue)
			htmlMsg = statMsg+'\n'
			logging.debug(statMsg)
			htmleNBFailureMsg += htmlMsg
hardy's avatar
hardy committed
1139 1140 1141 1142 1143
			#X2AP-PDU log
			statMsg = 'X2AP-PDU msg count =  '+str(x2ap_pdu)
			htmlMsg = statMsg+'\n'
			logging.debug(statMsg)
			htmleNBFailureMsg += htmlMsg
1144 1145 1146 1147 1148
			#nsa markers
			statMsg = 'logfile line count = ' + str(line_cnt)			
			htmlMsg = statMsg+'\n'
			logging.debug(statMsg)
			htmleNBFailureMsg += htmlMsg
1149 1150 1151 1152
			if len(nsa_markers['SgNBReleaseRequest'])!=0:
				statMsg = 'SgNBReleaseRequest = ' + str(len(nsa_markers['SgNBReleaseRequest'])) + ' occurences , starting line ' + str(nsa_markers['SgNBReleaseRequest'][0])
			else:
				statMsg = 'SgNBReleaseRequest = ' + str(len(nsa_markers['SgNBReleaseRequest'])) + ' occurences'
1153 1154 1155 1156 1157 1158 1159 1160 1161
			htmlMsg = statMsg+'\n'
			logging.debug(statMsg)
			htmleNBFailureMsg += htmlMsg
			statMsg = 'scgFailureInformationNR-r15 = ' + str(len(nsa_markers['scgFailureInformationNR-r15'])) + ' occurences'
			htmlMsg = statMsg+'\n'
			logging.debug(statMsg)
			htmleNBFailureMsg += htmlMsg			


Gabriele Perrone's avatar
Gabriele Perrone committed
1162 1163 1164
		if uciStatMsgCount > 0:
			statMsg = nodeB_prefix + 'NB showed ' + str(uciStatMsgCount) + ' "uci->stat" message(s)'
			logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
1165
			htmleNBFailureMsg += statMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
1166 1167 1168
		if pdcpFailure > 0:
			statMsg = nodeB_prefix + 'NB showed ' + str(pdcpFailure) + ' "PDCP Out of Resources" message(s)'
			logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
1169
			htmleNBFailureMsg += statMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
1170 1171 1172
		if ulschFailure > 0:
			statMsg = nodeB_prefix + 'NB showed ' + str(ulschFailure) + ' "ULSCH in error in round" message(s)'
			logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
1173
			htmleNBFailureMsg += statMsg + '\n'
1174 1175 1176 1177 1178 1179 1180 1181
		if ulschAllocateCCEerror > 0:
			statMsg = nodeB_prefix + 'NB showed ' + str(ulschAllocateCCEerror) + ' "eNB_dlsch_ulsch_scheduler(); ERROR ALLOCATING CCEs" message(s)'
			logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
			htmleNBFailureMsg += statMsg + '\n'
		if uplinkSegmentsAborted > 0:
			statMsg = nodeB_prefix + 'NB showed ' + str(uplinkSegmentsAborted) + ' "uplink segment error 0/2, aborted * segments" message(s)'
			logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
			htmleNBFailureMsg += statMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
1182 1183 1184
		if dropNotEnoughRBs > 0:
			statMsg = 'eNB showed ' + str(dropNotEnoughRBs) + ' "dropping, not enough RBs" message(s)'
			logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
1185
			htmleNBFailureMsg += statMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
1186 1187 1188
		if rrcSetupComplete > 0:
			rrcMsg = nodeB_prefix + 'NB completed ' + str(rrcSetupComplete) + ' RRC Connection Setup(s)'
			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
1189
			htmleNBFailureMsg += rrcMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
1190 1191
			rrcMsg = ' -- ' + str(rrcSetupComplete) + ' were completed'
			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
1192
			htmleNBFailureMsg += rrcMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
1193 1194 1195
		if rrcReleaseRequest > 0:
			rrcMsg = nodeB_prefix + 'NB requested ' + str(rrcReleaseRequest) + ' RRC Connection Release(s)'
			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
1196
			htmleNBFailureMsg += rrcMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
1197 1198 1199
		if rrcReconfigRequest > 0 or rrcReconfigComplete > 0:
			rrcMsg = nodeB_prefix + 'NB requested ' + str(rrcReconfigRequest) + ' RRC Connection Reconfiguration(s)'
			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
1200
			htmleNBFailureMsg += rrcMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
1201 1202
			rrcMsg = ' -- ' + str(rrcReconfigComplete) + ' were completed'
			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
1203
			htmleNBFailureMsg += rrcMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
1204 1205 1206
		if rrcReestablishRequest > 0 or rrcReestablishComplete > 0 or rrcReestablishReject > 0:
			rrcMsg = nodeB_prefix + 'NB requested ' + str(rrcReestablishRequest) + ' RRC Connection Reestablishment(s)'
			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
1207
			htmleNBFailureMsg += rrcMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
1208 1209
			rrcMsg = ' -- ' + str(rrcReestablishComplete) + ' were completed'
			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
1210
			htmleNBFailureMsg += rrcMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
1211 1212
			rrcMsg = ' -- ' + str(rrcReestablishReject) + ' were rejected'
			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
1213
			htmleNBFailureMsg += rrcMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
1214 1215 1216 1217
		if self.eNBmbmsEnables[int(self.eNB_instance)]:
			if mbmsRequestMsg > 0:
				rrcMsg = 'eNB requested ' + str(mbmsRequestMsg) + ' times the RLC for MBMS USER-PLANE'
				logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
1218
				htmleNBFailureMsg += rrcMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
1219 1220 1221
		if X2HO_inNbProcedures > 0:
			rrcMsg = 'eNB completed ' + str(X2HO_inNbProcedures) + ' X2 Handover Connection procedure(s)'
			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
1222
			htmleNBFailureMsg += rrcMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
1223 1224 1225
		if X2HO_outNbProcedures > 0:
			rrcMsg = 'eNB completed ' + str(X2HO_outNbProcedures) + ' X2 Handover Release procedure(s)'
			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
1226
			htmleNBFailureMsg += rrcMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
1227 1228 1229 1230 1231 1232
		if self.eNBOptions[int(self.eNB_instance)] != '':
			res1 = re.search('drx_Config_present prSetup', self.eNBOptions[int(self.eNB_instance)])
			if res1 is not None:
				if cdrxActivationMessageCount > 0:
					rrcMsg = 'eNB activated the CDRX Configuration for ' + str(cdrxActivationMessageCount) + ' time(s)'
					logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
1233
					htmleNBFailureMsg += rrcMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
1234 1235 1236
				else:
					rrcMsg = 'eNB did NOT ACTIVATE the CDRX Configuration'
					logging.debug('\u001B[1;37;43m ' + rrcMsg + ' \u001B[0m')
1237
					htmleNBFailureMsg += rrcMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
1238 1239 1240
		if rachCanceledProcedure > 0:
			rachMsg = nodeB_prefix + 'NB cancelled ' + str(rachCanceledProcedure) + ' RA procedure(s)'
			logging.debug('\u001B[1;30;43m ' + rachMsg + ' \u001B[0m')
1241
			htmleNBFailureMsg += rachMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
1242 1243 1244 1245 1246
		if isRRU:
			if isSlave:
				if slaveReceivesFrameResyncCmd:
					rruMsg = 'Slave RRU received the RRU_frame_resynch command from RAU'
					logging.debug('\u001B[1;30;43m ' + rruMsg + ' \u001B[0m')
1247
					htmleNBFailureMsg += rruMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
1248 1249 1250
				else:
					rruMsg = 'Slave RRU DID NOT receive the RRU_frame_resynch command from RAU'
					logging.debug('\u001B[1;37;41m ' + rruMsg + ' \u001B[0m')
1251
					htmleNBFailureMsg += rruMsg + '\n'
1252
					self.prematureExit = True
1253
					global_status = CONST.ENB_PROCESS_SLAVE_RRU_NOT_SYNCED
Gabriele Perrone's avatar
Gabriele Perrone committed
1254 1255
		if foundSegFault:
			logging.debug('\u001B[1;37;41m ' + nodeB_prefix + 'NB ended with a Segmentation Fault! \u001B[0m')
1256
			global_status = CONST.ENB_PROCESS_SEG_FAULT
Gabriele Perrone's avatar
Gabriele Perrone committed
1257 1258
		if foundAssertion:
			logging.debug('\u001B[1;37;41m ' + nodeB_prefix + 'NB ended with an assertion! \u001B[0m')
1259
			htmleNBFailureMsg += msgAssertion
1260
			global_status = CONST.ENB_PROCESS_ASSERTION
Gabriele Perrone's avatar
Gabriele Perrone committed
1261 1262
		if foundRealTimeIssue:
			logging.debug('\u001B[1;37;41m ' + nodeB_prefix + 'NB faced real time issues! \u001B[0m')
1263
			htmleNBFailureMsg += nodeB_prefix + 'NB faced real time issues! COUNT = '+ str(foundRealTimeIssue_cnt) +' lines\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
1264 1265 1266
		if rlcDiscardBuffer > 0:
			rlcMsg = nodeB_prefix + 'NB RLC discarded ' + str(rlcDiscardBuffer) + ' buffer(s)'
			logging.debug('\u001B[1;37;41m ' + rlcMsg + ' \u001B[0m')
1267
			htmleNBFailureMsg += rlcMsg + '\n'
1268
			global_status = CONST.ENB_PROCESS_REALTIME_ISSUE
Raphael Defosseux's avatar
Raphael Defosseux committed
1269
		HTML.htmleNBFailureMsg=htmleNBFailureMsg
1270
		# Runtime statistics for console output and HTML
1271 1272
		if runTime != '':
			logging.debug(runTime)
1273 1274 1275 1276
			logging.debug(userTime)
			logging.debug(systemTime)
			logging.debug(maxPhyMemUsage)
			logging.debug(nbContextSwitches)
1277
			self.runtime_stats='<pre>'+runTime + '\n'+ userTime + '\n' + systemTime + '\n' + maxPhyMemUsage + '\n' + nbContextSwitches+'</pre>'
1278
		return global_status