main.py 191 KB
Newer Older
1
# /*
2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20
# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
# * contributor license agreements.  See the NOTICE file distributed with
# * this work for additional information regarding copyright ownership.
# * The OpenAirInterface Software Alliance licenses this file to You under
# * the OAI Public License, Version 1.1  (the "License"); you may not use this file
# * except in compliance with the License.
# * You may obtain a copy of the License at
# *
# *      http://www.openairinterface.org/?page_id=698
# *
# * Unless required by applicable law or agreed to in writing, software
# * distributed under the License is distributed on an "AS IS" BASIS,
# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# * See the License for the specific language governing permissions and
# * limitations under the License.
# *-------------------------------------------------------------------------------
# * For more information about the OpenAirInterface (OAI) Software Alliance:
# *      contact@openairinterface.org
# */
21 22 23 24 25 26 27 28 29 30 31 32 33 34 35
#---------------------------------------------------------------------
# Python for CI of OAI-eNB + COTS-UE
#
#   Required Python Version
#     Python 3.x
#
#   Required Python Package
#     pexpect
#---------------------------------------------------------------------

#-----------------------------------------------------------
# Version
#-----------------------------------------------------------
Version = '0.1'

36 37 38 39 40 41 42 43 44
#-----------------------------------------------------------
# Constants
#-----------------------------------------------------------
ALL_PROCESSES_OK = 0
ENB_PROCESS_FAILED = -1
ENB_PROCESS_OK = +1
ENB_PROCESS_SEG_FAULT = -11
ENB_PROCESS_ASSERTION = -12
ENB_PROCESS_REALTIME_ISSUE = -13
45
ENB_PROCESS_NOLOGFILE_TO_ANALYZE = -14
46
ENB_PROCESS_SLAVE_RRU_NOT_SYNCED = -15
47 48 49 50 51 52 53
HSS_PROCESS_FAILED = -2
HSS_PROCESS_OK = +2
MME_PROCESS_FAILED = -3
MME_PROCESS_OK = +3
SPGW_PROCESS_FAILED = -4
SPGW_PROCESS_OK = +4
UE_IP_ADDRESS_ISSUE = -5
54 55 56
OAI_UE_PROCESS_NOLOGFILE_TO_ANALYZE = -20
OAI_UE_PROCESS_COULD_NOT_SYNC = -21
OAI_UE_PROCESS_ASSERTION = -22
57 58
OAI_UE_PROCESS_FAILED = -23
OAI_UE_PROCESS_NO_TUNNEL_INTERFACE = -24
59
OAI_UE_PROCESS_OK = +6
60

61 62 63 64 65
UE_STATUS_DETACHED = 0
UE_STATUS_DETACHING = 1
UE_STATUS_ATTACHING = 2
UE_STATUS_ATTACHED = 3

66 67 68 69 70 71 72 73
#-----------------------------------------------------------
# Import
#-----------------------------------------------------------
import sys		# arg
import re		# reg
import pexpect		# pexpect
import time		# sleep
import os
74
import subprocess
75 76 77 78
import xml.etree.ElementTree as ET
import logging
import datetime
import signal
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
79
from multiprocessing import Process, Lock, SimpleQueue
80
logging.basicConfig(
81 82
	level=logging.DEBUG,
	format="[%(asctime)s] %(name)s:%(levelname)s: %(message)s"
83 84 85 86 87 88 89
)

#-----------------------------------------------------------
# Class Declaration
#-----------------------------------------------------------
class SSHConnection():
	def __init__(self):
90 91 92 93 94
		self.ranRepository = ''
		self.ranBranch = ''
		self.ranAllowMerge = False
		self.ranCommitID = ''
		self.ranTargetBranch = ''
95 96 97 98 99 100 101
		self.eNBIPAddress = ''
		self.eNBUserName = ''
		self.eNBPassword = ''
		self.eNBSourceCodePath = ''
		self.EPCIPAddress = ''
		self.EPCUserName = ''
		self.EPCPassword = ''
102 103 104 105 106 107 108 109
		self.eNB1IPAddress = ''
		self.eNB1UserName = ''
		self.eNB1Password = ''
		self.eNB1SourceCodePath = ''
		self.eNB2IPAddress = ''
		self.eNB2UserName = ''
		self.eNB2Password = ''
		self.eNB2SourceCodePath = ''
110 111
		self.EPCSourceCodePath = ''
		self.EPCType = ''
112
		self.EPC_PcapFileName = ''
113 114 115 116
		self.ADBIPAddress = ''
		self.ADBUserName = ''
		self.ADBPassword = ''
		self.testCase_id = ''
117 118
		self.testXMLfiles = []
		self.nbTestXMLfiles = 0
119 120
		self.desc = ''
		self.Build_eNB_args = ''
121 122
		self.backgroundBuild = False
		self.backgroundBuildTestId = ['', '', '']
123
		self.Initialize_eNB_args = ''
124
		self.eNB_instance = ''
125
		self.eNB_serverId = ''
126 127
		self.eNBLogFiles = ['', '', '']
		self.eNBOptions = ['', '', '']
128 129 130 131
		self.ping_args = ''
		self.ping_packetloss_threshold = ''
		self.iperf_args = ''
		self.iperf_packetloss_threshold = ''
132
		self.iperf_profile = ''
133
		self.nbMaxUEtoAttach = -1
134
		self.UEDevices = []
135
		self.UEDevicesStatus = []
136
		self.CatMDevices = []
137
		self.UEIPAddresses = []
Raphael Defosseux's avatar
Raphael Defosseux committed
138 139 140
		self.htmlFile = ''
		self.htmlHeaderCreated = False
		self.htmlFooterCreated = False
141
		self.htmlUEConnected = -1
142
		self.htmleNBFailureMsg = ''
143
		self.htmlUEFailureMsg = ''
144
		self.picocom_closure = False
145 146 147 148 149
		self.idle_sleep_time = 0
		self.htmlTabRefs = []
		self.htmlTabNames = []
		self.htmlTabIcons = []
		self.finalStatus = False
150 151 152
		self.OsVersion = ''
		self.KernelVersion = ''
		self.UhdVersion = ''
153
		self.UsrpBoard = ''
154 155 156
		self.CpuNb = ''
		self.CpuModel = ''
		self.CpuMHz = ''
157 158 159
		self.UEIPAddress = ''
		self.UEUserName = ''
		self.UEPassword = ''
Boris Djalal's avatar
Boris Djalal committed
160
		self.UE_instance = ''
161
		self.UESourceCodePath = ''
162
		self.UELogFile = ''
Boris Djalal's avatar
Boris Djalal committed
163 164
		self.Build_OAI_UE_args = ''
		self.Initialize_OAI_UE_args = ''
165 166 167
		self.flexranCtrlInstalled = False
		self.flexranCtrlStarted = False
		self.expectedNbOfConnectedUEs = 0
168

169
	def open(self, ipaddress, username, password):
170 171 172 173 174
		count = 0
		connect_status = False
		while count < 4:
			self.ssh = pexpect.spawn('ssh', [username + '@' + ipaddress], timeout = 5)
			self.sshresponse = self.ssh.expect(['Are you sure you want to continue connecting (yes/no)?', 'password:', 'Last login', pexpect.EOF, pexpect.TIMEOUT])
175
			if self.sshresponse == 0:
176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201
				self.ssh.sendline('yes')
				self.ssh.expect('password:')
				self.ssh.sendline(password)
				self.sshresponse = self.ssh.expect(['\$', 'Permission denied', 'password:', pexpect.EOF, pexpect.TIMEOUT])
				if self.sshresponse == 0:
					count = 10
					connect_status = True
				else:
					logging.debug('self.sshresponse = ' + str(self.sshresponse))
			elif self.sshresponse == 1:
				self.ssh.sendline(password)
				self.sshresponse = self.ssh.expect(['\$', 'Permission denied', 'password:', pexpect.EOF, pexpect.TIMEOUT])
				if self.sshresponse == 0:
					count = 10
					connect_status = True
				else:
					logging.debug('self.sshresponse = ' + str(self.sshresponse))
			elif self.sshresponse == 2:
				# Checking if we are really on the remote client defined by its IP address
				self.command('stdbuf -o0 ifconfig | egrep --color=never "inet addr:"', '\$', 5)
				result = re.search(str(ipaddress), str(self.ssh.before))
				if result is None:
					self.close()
				else:
					count = 10
					connect_status = True
202
			else:
203 204
				# debug output
				logging.debug(str(self.ssh.before))
205
				logging.debug('self.sshresponse = ' + str(self.sshresponse))
Raphael Defosseux's avatar
Raphael Defosseux committed
206 207 208
			# adding a tempo when failure
			if not connect_status:
				time.sleep(1)
209 210
			count += 1
		if connect_status:
211 212
			pass
		else:
213
			sys.exit('SSH Connection Failed')
214

215 216 217 218 219 220
	def command(self, commandline, expectedline, timeout):
		logging.debug(commandline)
		self.ssh.timeout = timeout
		self.ssh.sendline(commandline)
		self.sshresponse = self.ssh.expect([expectedline, pexpect.EOF, pexpect.TIMEOUT])
		if self.sshresponse == 0:
Raphael Defosseux's avatar
Raphael Defosseux committed
221
			return 0
222 223 224 225 226 227 228
		elif self.sshresponse == 1:
			logging.debug('\u001B[1;37;41m Unexpected EOF \u001B[0m')
			logging.debug('Expected Line : ' + expectedline)
			sys.exit(self.sshresponse)
		elif self.sshresponse == 2:
			logging.debug('\u001B[1;37;41m Unexpected TIMEOUT \u001B[0m')
			logging.debug('Expected Line : ' + expectedline)
229
			result = re.search('ping |iperf |picocom', str(commandline))
Raphael Defosseux's avatar
Raphael Defosseux committed
230
			if result is None:
231
				logging.debug(str(self.ssh.before))
Raphael Defosseux's avatar
Raphael Defosseux committed
232 233 234
				sys.exit(self.sshresponse)
			else:
				return -1
235 236 237 238
		else:
			logging.debug('\u001B[1;37;41m Unexpected Others \u001B[0m')
			logging.debug('Expected Line : ' + expectedline)
			sys.exit(self.sshresponse)
239

240 241 242 243 244 245 246
	def close(self):
		self.ssh.timeout = 5
		self.ssh.sendline('exit')
		self.sshresponse = self.ssh.expect([pexpect.EOF, pexpect.TIMEOUT])
		if self.sshresponse == 0:
			pass
		elif self.sshresponse == 1:
247 248
			if not self.picocom_closure:
				logging.debug('\u001B[1;37;41m Unexpected TIMEOUT during closing\u001B[0m')
249
		else:
250
			logging.debug('\u001B[1;37;41m Unexpected Others during closing\u001B[0m')
251

252
	def copyin(self, ipaddress, username, password, source, destination):
253 254
		count = 0
		copy_status = False
Raphael Defosseux's avatar
Raphael Defosseux committed
255
		logging.debug('scp '+ username + '@' + ipaddress + ':' + source + ' ' + destination)
256
		while count < 10:
257
			scp_spawn = pexpect.spawn('scp '+ username + '@' + ipaddress + ':' + source + ' ' + destination, timeout = 100)
258
			scp_response = scp_spawn.expect(['Are you sure you want to continue connecting (yes/no)?', 'password:', pexpect.EOF, pexpect.TIMEOUT])
Raphael Defosseux's avatar
Raphael Defosseux committed
259
			if scp_response == 0:
260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279
				scp_spawn.sendline('yes')
				scp_spawn.expect('password:')
				scp_spawn.sendline(password)
				scp_response = scp_spawn.expect(['\$', 'Permission denied', 'password:', pexpect.EOF, pexpect.TIMEOUT])
				if scp_response == 0:
					count = 10
					copy_status = True
				else:
					logging.debug('1 - scp_response = ' + str(scp_response))
			elif scp_response == 1:
				scp_spawn.sendline(password)
				scp_response = scp_spawn.expect(['\$', 'Permission denied', 'password:', pexpect.EOF, pexpect.TIMEOUT])
				if scp_response == 0 or scp_response == 3:
					count = 10
					copy_status = True
				else:
					logging.debug('2 - scp_response = ' + str(scp_response))
			elif scp_response == 2:
				count = 10
				copy_status = True
Raphael Defosseux's avatar
Raphael Defosseux committed
280
			else:
281 282 283 284 285 286
				logging.debug('3 - scp_response = ' + str(scp_response))
			# adding a tempo when failure
			if not copy_status:
				time.sleep(1)
			count += 1
		if copy_status:
287
			return 0
Raphael Defosseux's avatar
Raphael Defosseux committed
288
		else:
289
			return -1
Raphael Defosseux's avatar
Raphael Defosseux committed
290

291
	def copyout(self, ipaddress, username, password, source, destination):
292 293
		count = 0
		copy_status = False
294
		logging.debug('scp ' + source + ' ' + username + '@' + ipaddress + ':' + destination)
295
		while count < 4:
296
			scp_spawn = pexpect.spawn('scp ' + source + ' ' + username + '@' + ipaddress + ':' + destination, timeout = 100)
297
			scp_response = scp_spawn.expect(['Are you sure you want to continue connecting (yes/no)?', 'password:', pexpect.EOF, pexpect.TIMEOUT])
298
			if scp_response == 0:
299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318
				scp_spawn.sendline('yes')
				scp_spawn.expect('password:')
				scp_spawn.sendline(password)
				scp_response = scp_spawn.expect(['\$', 'Permission denied', 'password:', pexpect.EOF, pexpect.TIMEOUT])
				if scp_response == 0:
					count = 10
					copy_status = True
				else:
					logging.debug('1 - scp_response = ' + str(scp_response))
			elif scp_response == 1:
				scp_spawn.sendline(password)
				scp_response = scp_spawn.expect(['\$', 'Permission denied', 'password:', pexpect.EOF, pexpect.TIMEOUT])
				if scp_response == 0 or scp_response == 3:
					count = 10
					copy_status = True
				else:
					logging.debug('2 - scp_response = ' + str(scp_response))
			elif scp_response == 2:
				count = 10
				copy_status = True
319
			else:
320 321 322 323 324 325
				logging.debug('3 - scp_response = ' + str(scp_response))
			# adding a tempo when failure
			if not copy_status:
				time.sleep(1)
			count += 1
		if copy_status:
326 327 328 329
			pass
		else:
			sys.exit('SCP failed')

330
	def BuildeNB(self):
331
		if self.ranRepository == '' or self.ranBranch == '' or self.ranCommitID == '':
332 333
			Usage()
			sys.exit('Insufficient Parameter')
334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355
		if self.eNB_serverId == '0':
			lIpAddr = self.eNBIPAddress
			lUserName = self.eNBUserName
			lPassWord = self.eNBPassword
			lSourcePath = self.eNBSourceCodePath
		elif self.eNB_serverId == '1':
			lIpAddr = self.eNB1IPAddress
			lUserName = self.eNB1UserName
			lPassWord = self.eNB1Password
			lSourcePath = self.eNB1SourceCodePath
		elif self.eNB_serverId == '2':
			lIpAddr = self.eNB2IPAddress
			lUserName = self.eNB2UserName
			lPassWord = self.eNB2Password
			lSourcePath = self.eNB2SourceCodePath
		if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '':
			Usage()
			sys.exit('Insufficient Parameter')
		self.open(lIpAddr, lUserName, lPassWord)
		self.command('mkdir -p ' + lSourcePath, '\$', 5)
		self.command('cd ' + lSourcePath, '\$', 5)
		self.command('if [ ! -e .git ]; then stdbuf -o0 git clone ' + self.ranRepository + ' .; else stdbuf -o0 git fetch; fi', '\$', 600)
356
		# Raphael: here add a check if git clone or git fetch went smoothly
357 358
		self.command('git config user.email "jenkins@openairinterface.org"', '\$', 5)
		self.command('git config user.name "OAI Jenkins"', '\$', 5)
359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389
		# Checking the BUILD INFO file
		if not self.backgroundBuild:
			self.command('ls *.txt', '\$', 5)
			result = re.search('LAST_BUILD_INFO', str(self.ssh.before))
			if result is not None:
				mismatch = False
				self.command('grep SRC_COMMIT LAST_BUILD_INFO.txt', '\$', 2)
				result = re.search(self.ranCommitID, str(self.ssh.before))
				if result is None:
					mismatch = True
				self.command('grep MERGED_W_TGT_BRANCH LAST_BUILD_INFO.txt', '\$', 2)
				if (self.ranAllowMerge):
					result = re.search('YES', str(self.ssh.before))
					if result is None:
						mismatch = True
					self.command('grep TGT_BRANCH LAST_BUILD_INFO.txt', '\$', 2)
					if self.ranTargetBranch == '':
						result = re.search('develop', str(self.ssh.before))
					else:
						result = re.search(self.ranTargetBranch, str(self.ssh.before))
					if result is None:
						mismatch = True
				else:
					result = re.search('NO', str(self.ssh.before))
					if result is None:
						mismatch = True
				if not mismatch:
					self.close()
					self.CreateHtmlTestRow(self.Build_eNB_args, 'OK', ALL_PROCESSES_OK)
					return
				
390
		self.command('echo ' + lPassWord + ' | sudo -S git clean -x -d -ff', '\$', 30)
391
		# if the commit ID is provided use it to point to it
392 393
		if self.ranCommitID != '':
			self.command('git checkout -f ' + self.ranCommitID, '\$', 5)
394 395
		# if the branch is not develop, then it is a merge request and we need to do 
		# the potential merge. Note that merge conflicts should already been checked earlier
396 397 398
		if (self.ranAllowMerge):
			if self.ranTargetBranch == '':
				if (self.ranBranch != 'develop') and (self.ranBranch != 'origin/develop'):
399 400
					self.command('git merge --ff origin/develop -m "Temporary merge for CI"', '\$', 5)
			else:
401 402
				logging.debug('Merging with the target branch: ' + self.ranTargetBranch)
				self.command('git merge --ff origin/' + self.ranTargetBranch + ' -m "Temporary merge for CI"', '\$', 5)
403 404
		self.command('source oaienv', '\$', 5)
		self.command('cd cmake_targets', '\$', 5)
405 406
		self.command('mkdir -p log', '\$', 5)
		self.command('chmod 777 log', '\$', 5)
407
		# no need to remove in log (git clean did the trick)
408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452
		if self.backgroundBuild:
			self.command('echo "./build_oai ' + self.Build_eNB_args + '" > ./my-lte-softmodem-build.sh', '\$', 5)
			self.command('chmod 775 ./my-lte-softmodem-build.sh', '\$', 5)
			self.command('echo ' + lPassWord + ' | sudo -S -E daemon --inherit --unsafe --name=build_enb_daemon --chdir=' + lSourcePath + '/cmake_targets -o ' + lSourcePath + '/cmake_targets/compile_oai_enb.log ./my-lte-softmodem-build.sh', '\$', 5)
			self.close()
			self.CreateHtmlTestRow(self.Build_eNB_args, 'OK', ALL_PROCESSES_OK)
			self.backgroundBuildTestId[int(self.eNB_instance)] = self.testCase_id
			return
		self.command('stdbuf -o0 ./build_oai ' + self.Build_eNB_args + ' 2>&1 | stdbuf -o0 tee compile_oai_enb.log', 'Bypassing the Tests|build have failed', 1500)
		self.checkBuildeNB(lIpAddr, lUserName, lPassWord, lSourcePath, self.testCase_id)

	def WaitBuildeNBisFinished(self):
		if self.eNB_serverId == '0':
			lIpAddr = self.eNBIPAddress
			lUserName = self.eNBUserName
			lPassWord = self.eNBPassword
			lSourcePath = self.eNBSourceCodePath
		elif self.eNB_serverId == '1':
			lIpAddr = self.eNB1IPAddress
			lUserName = self.eNB1UserName
			lPassWord = self.eNB1Password
			lSourcePath = self.eNB1SourceCodePath
		elif self.eNB_serverId == '2':
			lIpAddr = self.eNB2IPAddress
			lUserName = self.eNB2UserName
			lPassWord = self.eNB2Password
			lSourcePath = self.eNB2SourceCodePath
		if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '':
			Usage()
			sys.exit('Insufficient Parameter')
		self.open(lIpAddr, lUserName, lPassWord)
		count = 40
		buildOAIprocess = True
		while (count > 0) and buildOAIprocess:
			self.command('ps aux | grep --color=never build_ | grep -v grep', '\$', 3)
			result = re.search('build_oai', str(self.ssh.before))
			if result is None:
				buildOAIprocess = False
			else:
				count -= 1
				time.sleep(30)
		self.checkBuildeNB(lIpAddr, lUserName, lPassWord, lSourcePath, self.backgroundBuildTestId[int(self.eNB_instance)])

	def checkBuildeNB(self, lIpAddr, lUserName, lPassWord, lSourcePath, testcaseId):
		self.command('cd ' + lSourcePath + '/cmake_targets', '\$', 3)
453 454 455 456 457 458
		self.command('ls lte_build_oai/build', '\$', 3)
		self.command('ls lte_build_oai/build', '\$', 3)
		buildStatus = True
		result = re.search('lte-softmodem', str(self.ssh.before))
		if result is None:
			buildStatus = False
459 460 461 462 463 464 465
		else:
			# Generating a BUILD INFO file
			self.command('echo "SRC_BRANCH: ' + self.ranBranch + '" > ../LAST_BUILD_INFO.txt', '\$', 2)
			self.command('echo "SRC_COMMIT: ' + self.ranCommitID + '" >> ../LAST_BUILD_INFO.txt', '\$', 2)
			if (self.ranAllowMerge):
				self.command('echo "MERGED_W_TGT_BRANCH: YES" >> ../LAST_BUILD_INFO.txt', '\$', 2)
				if self.ranTargetBranch == '':
466
					self.command('echo "TGT_BRANCH: develop" >> ../LAST_BUILD_INFO.txt', '\$', 2)
467
				else:
468
					self.command('echo "TGT_BRANCH: ' + self.ranBranch + '" >> ../LAST_BUILD_INFO.txt', '\$', 2)
469 470
			else:
				self.command('echo "MERGED_W_TGT_BRANCH: NO" >> ../LAST_BUILD_INFO.txt', '\$', 2)
471 472 473
		self.command('mkdir -p build_log_' + testcaseId, '\$', 5)
		self.command('mv log/* ' + 'build_log_' + testcaseId, '\$', 5)
		self.command('mv compile_oai_enb.log ' + 'build_log_' + testcaseId, '\$', 5)
474 475
		if self.eNB_serverId != '0':
			self.command('cd cmake_targets', '\$', 5)
476 477
			self.command('if [ -e tmp_build' + testcaseId + '.zip ]; then rm -f tmp_build' + testcaseId + '.zip; fi', '\$', 5)
			self.command('zip -r -qq tmp_build' + testcaseId + '.zip build_log_' + testcaseId, '\$', 5)
478
			self.close()
479 480 481 482 483 484
			if (os.path.isfile('./tmp_build' + testcaseId + '.zip')):
				os.remove('./tmp_build' + testcaseId + '.zip')
			self.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/tmp_build' + testcaseId + '.zip', '.')
			if (os.path.isfile('./tmp_build' + testcaseId + '.zip')):
				self.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, './tmp_build' + testcaseId + '.zip', self.eNBSourceCodePath + '/cmake_targets/.')
				os.remove('./tmp_build' + testcaseId + '.zip')
485 486
				self.open(self.eNBIPAddress, self.eNBUserName, self.eNBPassword)
				self.command('cd ' + self.eNBSourceCodePath + '/cmake_targets', '\$', 5)
487 488
				self.command('unzip -qq -DD tmp_build' + testcaseId + '.zip', '\$', 5)
				self.command('rm -f tmp_build' + testcaseId + '.zip', '\$', 5)
489 490 491 492
				self.close()
		else:
			self.close()

493 494 495 496 497 498 499
		if buildStatus:
			self.CreateHtmlTestRow(self.Build_eNB_args, 'OK', ALL_PROCESSES_OK)
		else:
			logging.error('\u001B[1m Building OAI eNB Failed\u001B[0m')
			self.CreateHtmlTestRow(self.Build_eNB_args, 'KO', ALL_PROCESSES_OK)
			self.CreateHtmlTabFooter(False)
			sys.exit(1)
500

501
	def BuildOAIUE(self):
502
		if self.UEIPAddress == '' or self.ranRepository == '' or self.ranBranch == '' or self.UEUserName == '' or self.UEPassword == '' or self.UESourceCodePath == '':
503 504 505
			Usage()
			sys.exit('Insufficient Parameter')
		self.open(self.UEIPAddress, self.UEUserName, self.UEPassword)
506 507
		self.command('mkdir -p ' + self.UESourceCodePath, '\$', 5)
		self.command('cd ' + self.UESourceCodePath, '\$', 5)
508
		self.command('if [ ! -e .git ]; then stdbuf -o0 git clone ' + self.ranRepository + ' .; else stdbuf -o0 git fetch; fi', '\$', 600)
509 510 511 512 513
		# here add a check if git clone or git fetch went smoothly
		self.command('git config user.email "jenkins@openairinterface.org"', '\$', 5)
		self.command('git config user.name "OAI Jenkins"', '\$', 5)
		self.command('echo ' + self.UEPassword + ' | sudo -S git clean -x -d -ff', '\$', 30)
		# if the commit ID is provided use it to point to it
514 515
		if self.ranCommitID != '':
			self.command('git checkout -f ' + self.ranCommitID, '\$', 5)
516 517
		# if the branch is not develop, then it is a merge request and we need to do 
		# the potential merge. Note that merge conflicts should already been checked earlier
518 519 520
		if (self.ranAllowMerge):
			if self.ranTargetBranch == '':
				if (self.ranBranch != 'develop') and (self.ranBranch != 'origin/develop'):
521 522
					self.command('git merge --ff origin/develop -m "Temporary merge for CI"', '\$', 5)
			else:
523 524
				logging.debug('Merging with the target branch: ' + self.ranTargetBranch)
				self.command('git merge --ff origin/' + self.ranTargetBranch + ' -m "Temporary merge for CI"', '\$', 5)
525 526 527 528 529
		self.command('source oaienv', '\$', 5)
		self.command('cd cmake_targets', '\$', 5)
		self.command('mkdir -p log', '\$', 5)
		self.command('chmod 777 log', '\$', 5)
		# no need to remove in log (git clean did the trick)
530 531 532 533 534 535 536
		self.command('stdbuf -o0 ./build_oai ' + self.Build_OAI_UE_args + ' 2>&1 | stdbuf -o0 tee compile_oai_ue.log', 'Bypassing the Tests|build have failed', 600)
		self.command('ls lte_build_oai/build', '\$', 3)
		self.command('ls lte_build_oai/build', '\$', 3)
		buildStatus = True
		result = re.search('lte-uesoftmodem', str(self.ssh.before))
		if result is None:
			buildStatus = False
537 538 539 540
		self.command('mkdir -p build_log_' + self.testCase_id, '\$', 5)
		self.command('mv log/* ' + 'build_log_' + self.testCase_id, '\$', 5)
		self.command('mv compile_oai_ue.log ' + 'build_log_' + self.testCase_id, '\$', 5)
		self.close()
541 542 543 544 545 546 547
		if buildStatus:
			self.CreateHtmlTestRow(self.Build_OAI_UE_args, 'OK', ALL_PROCESSES_OK, 'OAI UE')
		else:
			logging.error('\u001B[1m Building OAI UE Failed\u001B[0m')
			self.CreateHtmlTestRow(self.Build_OAI_UE_args, 'KO', ALL_PROCESSES_OK, 'OAI UE')
			self.CreateHtmlTabFooter(False)
			sys.exit(1)
548

549 550 551 552 553 554 555
	def InitializeHSS(self):
		if self.EPCIPAddress == '' or self.EPCUserName == '' or self.EPCPassword == '' or self.EPCSourceCodePath == '' or self.EPCType == '':
			Usage()
			sys.exit('Insufficient Parameter')
		self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
		if re.match('OAI', self.EPCType, re.IGNORECASE):
			logging.debug('Using the OAI EPC HSS')
556
			self.command('cd ' + self.EPCSourceCodePath, '\$', 5)
557 558
			self.command('source oaienv', '\$', 5)
			self.command('cd scripts', '\$', 5)
559
			self.command('echo ' + self.EPCPassword + ' | sudo -S ./run_hss 2>&1 | stdbuf -o0 awk \'{ print strftime("[%Y/%m/%d %H:%M:%S] ",systime()) $0 }\' | stdbuf -o0 tee -a hss_' + self.testCase_id + '.log &', 'Core state: 2 -> 3', 35)
560 561
		else:
			logging.debug('Using the ltebox simulated HSS')
562 563 564 565
			self.command('if [ -d ' + self.EPCSourceCodePath + '/scripts ]; then echo ' + self.eNBPassword + ' | sudo -S rm -Rf ' + self.EPCSourceCodePath + '/scripts ; fi', '\$', 5)
			self.command('mkdir -p ' + self.EPCSourceCodePath + '/scripts', '\$', 5)
			self.command('cd /opt/hss_sim0609', '\$', 5)
			self.command('echo ' + self.EPCPassword + ' | sudo -S rm -f hss.log daemon.log', '\$', 5)
566 567
			self.command('echo ' + self.EPCPassword + ' | sudo -S echo "Starting sudo session" && sudo daemon --unsafe --name=simulated_hss --chdir=/opt/hss_sim0609 ./starthss_real  ', '\$', 5)
		self.close()
568
		self.CreateHtmlTestRow(self.EPCType, 'OK', ALL_PROCESSES_OK)
569

570 571 572 573 574 575
	def InitializeMME(self):
		if self.EPCIPAddress == '' or self.EPCUserName == '' or self.EPCPassword == '' or self.EPCSourceCodePath == '' or self.EPCType == '':
			Usage()
			sys.exit('Insufficient Parameter')
		self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
		if re.match('OAI', self.EPCType, re.IGNORECASE):
576
			self.command('cd ' + self.EPCSourceCodePath, '\$', 5)
577 578
			self.command('source oaienv', '\$', 5)
			self.command('cd scripts', '\$', 5)
579
			self.command('stdbuf -o0 hostname', '\$', 5)
580 581 582 583 584
			result = re.search('hostname\\\\r\\\\n(?P<host_name>[a-zA-Z0-9\-\_]+)\\\\r\\\\n', str(self.ssh.before))
			if result is None:
				logging.debug('\u001B[1;37;41m Hostname Not Found! \u001B[0m')
				sys.exit(1)
			host_name = result.group('host_name')
585
			self.command('echo ' + self.EPCPassword + ' | sudo -S ./run_mme 2>&1 | stdbuf -o0 tee -a mme_' + self.testCase_id + '.log &', 'MME app initialization complete', 100)
586
		else:
587
			self.command('cd /opt/ltebox/tools', '\$', 5)
588 589
			self.command('echo ' + self.EPCPassword + ' | sudo -S ./start_mme', '\$', 5)
		self.close()
590
		self.CreateHtmlTestRow(self.EPCType, 'OK', ALL_PROCESSES_OK)
591 592 593

	def InitializeSPGW(self):
		if self.EPCIPAddress == '' or self.EPCUserName == '' or self.EPCPassword == '' or self.EPCSourceCodePath == '' or self.EPCType == '':
594 595
			Usage()
			sys.exit('Insufficient Parameter')
596 597
		self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
		if re.match('OAI', self.EPCType, re.IGNORECASE):
598
			self.command('cd ' + self.EPCSourceCodePath, '\$', 5)
599 600
			self.command('source oaienv', '\$', 5)
			self.command('cd scripts', '\$', 5)
601
			self.command('echo ' + self.EPCPassword + ' | sudo -S ./run_spgw 2>&1 | stdbuf -o0 tee -a spgw_' + self.testCase_id + '.log &', 'Initializing SPGW-APP task interface: DONE', 30)
602
		else:
603
			self.command('cd /opt/ltebox/tools', '\$', 5)
604 605
			self.command('echo ' + self.EPCPassword + ' | sudo -S ./start_xGw', '\$', 5)
		self.close()
606
		self.CreateHtmlTestRow(self.EPCType, 'OK', ALL_PROCESSES_OK)
607

608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638
	def CheckFlexranCtrlInstallation(self):
		if self.EPCIPAddress == '' or self.EPCUserName == '' or self.EPCPassword == '':
			return
		self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
		self.command('ls -ls /opt/flexran_rtc/*/rt_controller', '\$', 5)
		result = re.search('/opt/flexran_rtc/build/rt_controller', str(self.ssh.before))
		if result is not None:
			self.flexranCtrlInstalled = True
			logging.debug('Flexran Controller is installed')
		self.close()

	def InitializeFlexranCtrl(self):
		if self.flexranCtrlInstalled == False:
			return
		if self.EPCIPAddress == '' or self.EPCUserName == '' or self.EPCPassword == '':
			Usage()
			sys.exit('Insufficient Parameter')
		self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
		self.command('cd /opt/flexran_rtc', '\$', 5)
		self.command('echo ' + self.EPCPassword + ' | sudo -S rm -f log/*.log', '\$', 5)
		self.command('echo ' + self.EPCPassword + ' | sudo -S echo "build/rt_controller -c log_config/basic_log" > ./my-flexran-ctl.sh', '\$', 5)
		self.command('echo ' + self.EPCPassword + ' | sudo -S chmod 755 ./my-flexran-ctl.sh', '\$', 5)
		self.command('echo ' + self.EPCPassword + ' | sudo -S daemon --unsafe --name=flexran_rtc_daemon --chdir=/opt/flexran_rtc -o /opt/flexran_rtc/log/flexranctl_' + self.testCase_id + '.log ././my-flexran-ctl.sh', '\$', 5)
		self.command('ps -aux | grep --color=never rt_controller', '\$', 5)
		result = re.search('rt_controller -c ', str(self.ssh.before))
		if result is not None:
			logging.debug('\u001B[1m Initialize FlexRan Controller Completed\u001B[0m')
			self.flexranCtrlStarted = True
		self.close()
		self.CreateHtmlTestRow('N/A', 'OK', ALL_PROCESSES_OK)

639
	def InitializeeNB(self):
640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655
		if self.eNB_serverId == '0':
			lIpAddr = self.eNBIPAddress
			lUserName = self.eNBUserName
			lPassWord = self.eNBPassword
			lSourcePath = self.eNBSourceCodePath
		elif self.eNB_serverId == '1':
			lIpAddr = self.eNB1IPAddress
			lUserName = self.eNB1UserName
			lPassWord = self.eNB1Password
			lSourcePath = self.eNB1SourceCodePath
		elif self.eNB_serverId == '2':
			lIpAddr = self.eNB2IPAddress
			lUserName = self.eNB2UserName
			lPassWord = self.eNB2Password
			lSourcePath = self.eNB2SourceCodePath
		if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '':
656 657
			Usage()
			sys.exit('Insufficient Parameter')
658 659 660 661 662 663 664
		check_eNB = False
		check_OAI_UE = False
		pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE)
		if (pStatus < 0):
			self.CreateHtmlTestRow(self.Initialize_eNB_args, 'KO', pStatus)
			self.CreateHtmlTabFooter(False)
			sys.exit(1)
665 666 667 668 669 670 671 672 673
		# If tracer options is on, running tshark on EPC side and capture traffic b/ EPC and eNB
		result = re.search('T_stdout', str(self.Initialize_eNB_args))
		if result is not None:
			self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
			self.command('ip addr show | awk -f /tmp/active_net_interfaces.awk | egrep -v "lo|tun"', '\$', 5)
			result = re.search('interfaceToUse=(?P<eth_interface>[a-zA-Z0-9\-\_]+)done', str(self.ssh.before))
			if result is not None:
				eth_interface = result.group('eth_interface')
				logging.debug('\u001B[1m Launching tshark on interface ' + eth_interface + '\u001B[0m')
674
				self.EPC_PcapFileName = 'enb_' + self.testCase_id + '_s1log.pcap'
675
				self.command('echo ' + self.EPCPassword + ' | sudo -S rm -f /tmp/' + self.EPC_PcapFileName, '\$', 5)
676
				self.command('echo $USER; nohup sudo tshark -f "host ' + lIpAddr +'" -i ' + eth_interface + ' -w /tmp/' + self.EPC_PcapFileName + ' > /tmp/tshark.log 2>&1 &', self.EPCUserName, 5)
677
			self.close()
678 679
		self.open(lIpAddr, lUserName, lPassWord)
		self.command('cd ' + lSourcePath, '\$', 5)
680 681
		# Initialize_eNB_args usually start with -O and followed by the location in repository
		full_config_file = self.Initialize_eNB_args.replace('-O ','')
682
		extra_options = ''
683 684 685
		extIdx = full_config_file.find('.conf')
		if (extIdx > 0):
			extra_options = full_config_file[extIdx + 5:]
686 687 688 689 690 691
			# if tracer options is on, compiling and running T Tracer
			result = re.search('T_stdout', str(extra_options))
			if result is not None:
				logging.debug('\u001B[1m Compiling and launching T Tracer\u001B[0m')
				self.command('cd common/utils/T/tracer', '\$', 5)
				self.command('make', '\$', 10)
692 693
				self.command('echo $USER; nohup ./record -d ../T_messages.txt -o ' + lSourcePath + '/cmake_targets/enb_' + self.testCase_id + '_record.raw -ON -off VCD -off HEAVY -off LEGACY_GROUP_TRACE -off LEGACY_GROUP_DEBUG > ' + lSourcePath + '/cmake_targets/enb_' + self.testCase_id + '_record.log 2>&1 &', lUserName, 5)
				self.command('cd ' + lSourcePath, '\$', 5)
694 695 696 697
			full_config_file = full_config_file[:extIdx + 5]
			config_path, config_file = os.path.split(full_config_file)
		else:
			sys.exit('Insufficient Parameter')
698
		ci_full_config_file = config_path + '/ci-' + config_file
699
		rruCheck = False
700
		result = re.search('^rru|^rcc|^du.band', str(config_file))
701 702
		if result is not None:
			rruCheck = True
Raphael Defosseux's avatar
Raphael Defosseux committed
703
		# do not reset board twice in IF4.5 case
704
		result = re.search('^rru|^enb|^du.band', str(config_file))
Raphael Defosseux's avatar
Raphael Defosseux committed
705
		if result is not None:
706
			self.command('echo ' + lPassWord + ' | sudo -S uhd_find_devices', '\$', 10)
Raphael Defosseux's avatar
Raphael Defosseux committed
707 708 709
			result = re.search('type: b200', str(self.ssh.before))
			if result is not None:
				logging.debug('Found a B2xx device --> resetting it')
710
				self.command('echo ' + lPassWord + ' | sudo -S b2xx_fx3_utils --reset-device', '\$', 10)
Raphael Defosseux's avatar
Raphael Defosseux committed
711
				# Reloading FGPA bin firmware
712
				self.command('echo ' + lPassWord + ' | sudo -S uhd_find_devices', '\$', 15)
713 714
		# Make a copy and adapt to EPC / eNB IP addresses
		self.command('cp ' + full_config_file + ' ' + ci_full_config_file, '\$', 5)
715
		self.command('sed -i -e \'s/CI_MME_IP_ADDR/' + self.EPCIPAddress + '/\' ' + ci_full_config_file, '\$', 2);
716 717 718 719
		self.command('sed -i -e \'s/CI_ENB_IP_ADDR/' + lIpAddr + '/\' ' + ci_full_config_file, '\$', 2);
		self.command('sed -i -e \'s/CI_RCC_IP_ADDR/' + self.eNBIPAddress + '/\' ' + ci_full_config_file, '\$', 2);
		self.command('sed -i -e \'s/CI_RRU1_IP_ADDR/' + self.eNB1IPAddress + '/\' ' + ci_full_config_file, '\$', 2);
		self.command('sed -i -e \'s/CI_RRU2_IP_ADDR/' + self.eNB2IPAddress + '/\' ' + ci_full_config_file, '\$', 2);
720 721 722
		if self.flexranCtrlInstalled and self.flexranCtrlStarted:
			self.command('sed -i -e \'s/FLEXRAN_ENABLED.*;/FLEXRAN_ENABLED        = "yes";/\' ' + ci_full_config_file, '\$', 2);
		else:
723
			self.command('sed -i -e \'s/FLEXRAN_ENABLED.*;/FLEXRAN_ENABLED        = "no";/\' ' + ci_full_config_file, '\$', 2);
724
		# Launch eNB with the modified config file
725 726
		self.command('source oaienv', '\$', 5)
		self.command('cd cmake_targets', '\$', 5)
727
		self.command('echo "ulimit -c unlimited && ./lte_build_oai/build/lte-softmodem -O ' + lSourcePath + '/' + ci_full_config_file + extra_options + '" > ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5)
Boris Djalal's avatar
Boris Djalal committed
728
		self.command('chmod 775 ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5)
729 730 731 732 733
		self.command('echo ' + lPassWord + ' | sudo -S rm -Rf enb_' + self.testCase_id + '.log', '\$', 5)
		self.command('echo ' + lPassWord + ' | sudo -S -E daemon --inherit --unsafe --name=enb' + str(self.eNB_instance) + '_daemon --chdir=' + lSourcePath + '/cmake_targets -o ' + lSourcePath + '/cmake_targets/enb_' + self.testCase_id + '.log ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5)
		self.eNBLogFiles[int(self.eNB_instance)] = 'enb_' + self.testCase_id + '.log'
		if extra_options != '':
			self.eNBOptions[int(self.eNB_instance)] = extra_options
734 735 736 737 738 739
		time.sleep(6)
		doLoop = True
		loopCounter = 10
		while (doLoop):
			loopCounter = loopCounter - 1
			if (loopCounter == 0):
740 741 742 743 744
				# In case of T tracer recording, we may need to kill it
				result = re.search('T_stdout', str(self.Initialize_eNB_args))
				if result is not None:
					self.command('killall --signal SIGKILL record', '\$', 5)
				self.close()
745
				doLoop = False
746
				logging.error('\u001B[1;37;41m eNB logging system did not show got sync! \u001B[0m')
747
				self.CreateHtmlTestRow('-O ' + config_file + extra_options, 'KO', ALL_PROCESSES_OK)
748
				self.CreateHtmlTabFooter(False)
749 750 751 752 753 754
				# In case of T tracer recording, we need to kill tshark on EPC side
				result = re.search('T_stdout', str(self.Initialize_eNB_args))
				if result is not None:
					self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
					logging.debug('\u001B[1m Stopping tshark \u001B[0m')
					self.command('echo ' + self.EPCPassword + ' | sudo -S killall --signal SIGKILL tshark', '\$', 5)
755
					if self.EPC_PcapFileName != '':
756
						time.sleep(0.5)
757
						self.command('echo ' + self.EPCPassword + ' | sudo -S chmod 666 /tmp/' + self.EPC_PcapFileName, '\$', 5)
758 759
					self.close()
					time.sleep(1)
760 761 762
					if self.EPC_PcapFileName != '':
						copyin_res = self.copyin(self.EPCIPAddress, self.EPCUserName, self.EPCPassword, '/tmp/' + self.EPC_PcapFileName, '.')
						if (copyin_res == 0):
763
							self.copyout(lIpAddr, lUserName, lPassWord, self.EPC_PcapFileName, lSourcePath + '/cmake_targets/.')
764
				sys.exit(1)
765
			else:
766
				self.command('stdbuf -o0 cat enb_' + self.testCase_id + '.log | egrep --text --color=never -i "wait|sync|Starting"', '\$', 4)
767 768 769
				if rruCheck:
					result = re.search('wait RUs', str(self.ssh.before))
				else:
770
					result = re.search('got sync|Starting F1AP at CU', str(self.ssh.before))
Raphael Defosseux's avatar
Raphael Defosseux committed
771 772 773 774
				if result is None:
					time.sleep(6)
				else:
					doLoop = False
775
					self.CreateHtmlTestRow('-O ' + config_file + extra_options, 'OK', ALL_PROCESSES_OK)
Raphael Defosseux's avatar
Raphael Defosseux committed
776
					logging.debug('\u001B[1m Initialize eNB Completed\u001B[0m')
777
					time.sleep(10)
778

779 780
		self.close()

781 782 783 784
	def InitializeUE_common(self, device_id):
		logging.debug('send adb commands')
		try:
			self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
785 786 787
			# enable data service
			self.command('stdbuf -o0 adb -s ' + device_id + ' shell svc data enable', '\$', 60)

788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805
			# The following commands are deprecated since we no longer work on Android 7+
			# self.command('stdbuf -o0 adb -s ' + device_id + ' shell settings put global airplane_mode_on 1', '\$', 10)
			# self.command('stdbuf -o0 adb -s ' + device_id + ' shell am broadcast -a android.intent.action.AIRPLANE_MODE --ez state true', '\$', 60)
			# a dedicated script has to be installed inside the UE
			# airplane mode on means call /data/local/tmp/off
			self.command('stdbuf -o0 adb -s ' + device_id + ' shell /data/local/tmp/off', '\$', 60)
			#airplane mode off means call /data/local/tmp/on
			logging.debug('\u001B[1mUE (' + device_id + ') Initialize Completed\u001B[0m')
			self.close()
		except:
			os.kill(os.getppid(),signal.SIGUSR1)

	def InitializeUE(self):
		if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '':
			Usage()
			sys.exit('Insufficient Parameter')
		multi_jobs = []
		for device_id in self.UEDevices:
806
			p = Process(target = self.InitializeUE_common, args = (device_id,))
807 808 809 810 811
			p.daemon = True
			p.start()
			multi_jobs.append(p)
		for job in multi_jobs:
			job.join()
812
		self.CreateHtmlTestRow('N/A', 'OK', ALL_PROCESSES_OK)
813

814 815 816 817
	def InitializeOAIUE(self):
		if self.UEIPAddress == '' or self.UEUserName == '' or self.UEPassword == '' or self.UESourceCodePath == '':
			Usage()
			sys.exit('Insufficient Parameter')
818 819 820 821 822 823 824 825 826
		result = re.search('--no-L2-connect', str(self.Initialize_OAI_UE_args))
		if result is None:
			check_eNB = True
			check_OAI_UE = False
			pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE)
			if (pStatus < 0):
				self.CreateHtmlTestRow(self.Initialize_OAI_UE_args, 'KO', pStatus)
				self.CreateHtmlTabFooter(False)
				sys.exit(1)
827
		self.open(self.UEIPAddress, self.UEUserName, self.UEPassword)
828
		# b2xx_fx3_utils reset procedure
829
		self.command('echo ' + self.UEPassword + ' | sudo -S uhd_find_devices', '\$', 10)
830 831 832
		result = re.search('type: b200', str(self.ssh.before))
		if result is not None:
			logging.debug('Found a B2xx device --> resetting it')
833
			self.command('echo ' + self.UEPassword + ' | sudo -S b2xx_fx3_utils --reset-device', '\$', 10)
834
			# Reloading FGPA bin firmware
835
			self.command('echo ' + self.UEPassword + ' | sudo -S uhd_find_devices', '\$', 15)
836
		else:
837
			logging.debug('Did not find any B2xx device')
838
		self.command('cd ' + self.UESourceCodePath, '\$', 5)
839
		self.command('source oaienv', '\$', 5)
Boris Djalal's avatar
Boris Djalal committed
840
		self.command('cd cmake_targets/lte_build_oai/build', '\$', 5)
841 842 843 844 845 846 847
		result = re.search('--no-L2-connect', str(self.Initialize_OAI_UE_args))
		# We may have to regenerate the .u* files
		if result is None:
			self.command('sed -e "s#93#92#" -e "s#8baf473f2f8fd09487cccbd7097c6862#fec86ba6eb707ed08905757b1bb44b8f#" -e "s#e734f8734007d6c5ce7a0508809e7e9c#C42449363BBAD02B66D16BC975D77CC1#" ../../../openair3/NAS/TOOLS/ue_eurecom_test_sfr.conf > ../../../openair3/NAS/TOOLS/ci-ue_eurecom_test_sfr.conf', '\$', 5)
			self.command('echo ' + self.UEPassword + ' | sudo -S rm -Rf .u*', '\$', 5)
			self.command('echo ' + self.UEPassword + ' | sudo -S ../../../targets/bin/conf2uedata -c ../../../openair3/NAS/TOOLS/ci-ue_eurecom_test_sfr.conf -o .', '\$', 5)
		# Launch UE with the modified config file
Boris Djalal's avatar
Boris Djalal committed
848 849 850
		self.command('echo "ulimit -c unlimited && ./lte-uesoftmodem ' + self.Initialize_OAI_UE_args + '" > ./my-lte-uesoftmodem-run' + str(self.UE_instance) + '.sh', '\$', 5)
		self.command('chmod 775 ./my-lte-uesoftmodem-run' + str(self.UE_instance) + '.sh', '\$', 5)
		self.UELogFile = 'ue_' + self.testCase_id + '.log'
851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874

		# We are now looping several times to hope we really sync w/ an eNB
		doOutterLoop = True
		outterLoopCounter = 5
		gotSyncStatus = True
		fullSyncStatus = True
		while (doOutterLoop):
			self.command('cd ' + self.UESourceCodePath + '/cmake_targets/lte_build_oai/build', '\$', 5)
			self.command('echo ' + self.UEPassword + ' | sudo -S rm -Rf ' + self.UESourceCodePath + '/cmake_targets/ue_' + self.testCase_id + '.log', '\$', 5)
			self.command('echo ' + self.UEPassword + ' | sudo -S -E daemon --inherit --unsafe --name=ue' + str(self.UE_instance) + '_daemon --chdir=' + self.UESourceCodePath + '/cmake_targets/lte_build_oai/build -o ' + self.UESourceCodePath + '/cmake_targets/ue_' + self.testCase_id + '.log ./my-lte-uesoftmodem-run' + str(self.UE_instance) + '.sh', '\$', 5)
			time.sleep(6)
			self.command('cd ../..', '\$', 5)
			doLoop = True
			loopCounter = 10
			gotSyncStatus = True
			# the 'got sync' message is for the UE threads synchronization
			while (doLoop):
				loopCounter = loopCounter - 1
				if (loopCounter == 0):
					# Here should never occur
					logging.error('"got sync" message never showed!')
					gotSyncStatus = False
					doLoop = False
					continue
Boris Djalal's avatar
Boris Djalal committed
875 876
				self.command('stdbuf -o0 cat ue_' + self.testCase_id + '.log | egrep --text --color=never -i "wait|sync"', '\$', 4)
				result = re.search('got sync', str(self.ssh.before))
877 878 879 880
				if result is None:
					time.sleep(6)
				else:
					doLoop = False
881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924
					logging.debug('Found "got sync" message!')
			if gotSyncStatus == False:
				# we certainly need to stop the lte-uesoftmodem process if it is still running!
				self.command('ps -aux | grep --text --color=never softmodem | grep -v grep', '\$', 4)
				result = re.search('lte-uesoftmodem', str(self.ssh.before))
				if result is not None:
					self.command('echo ' + self.UEPassword + ' | sudo -S killall --signal=SIGINT lte-uesoftmodem', '\$', 4)
					time.sleep(3)
			# We are now checking if sync w/ eNB DOES NOT OCCUR
			# Usually during the cell synchronization stage, the UE returns with No cell synchronization message
			doLoop = True
			loopCounter = 10
			while (doLoop):
				loopCounter = loopCounter - 1
				if (loopCounter == 0):
					# Here we do have a great chance that the UE did cell-sync w/ eNB
					doLoop = False
					doOutterLoop = False
					fullSyncStatus = True
					continue
				self.command('stdbuf -o0 cat ue_' + self.testCase_id + '.log | egrep --text --color=never -i "wait|sync"', '\$', 4)
				result = re.search('No cell synchronization found', str(self.ssh.before))
				if result is None:
					time.sleep(6)
				else:
					doLoop = False
					fullSyncStatus = False
					logging.debug('Found: "No cell synchronization" message! --> try again')
					time.sleep(6)
					self.command('ps -aux | grep --text --color=never softmodem | grep -v grep', '\$', 4)
					result = re.search('lte-uesoftmodem', str(self.ssh.before))
					if result is not None:
						self.command('echo ' + self.UEPassword + ' | sudo -S killall --signal=SIGINT lte-uesoftmodem', '\$', 4)
			outterLoopCounter = outterLoopCounter - 1
			if (outterLoopCounter == 0):
				doOutterLoop = False

		if fullSyncStatus and gotSyncStatus:
			result = re.search('--no-L2-connect', str(self.Initialize_OAI_UE_args))
			if result is None:
				self.command('ifconfig oaitun_ue1', '\$', 4)
				result = re.search('inet addr', str(self.ssh.before))
				if result is not None:
					logging.debug('\u001B[1m oaitun_ue1 interface is mounted and configured\u001B[0m')
925
					tunnelInterfaceStatus = True
926 927
				else:
					logging.error('\u001B[1m oaitun_ue1 interface is either NOT mounted or NOT configured\u001B[0m')
928 929 930
					tunnelInterfaceStatus = False
			else:
				tunnelInterfaceStatus = True
931

932
		self.close()
933 934 935 936 937 938 939 940 941 942
		if fullSyncStatus and gotSyncStatus and tunnelInterfaceStatus:
			self.CreateHtmlTestRow(self.Initialize_OAI_UE_args, 'OK', ALL_PROCESSES_OK, 'OAI UE')
			logging.debug('\u001B[1m Initialize OAI UE Completed\u001B[0m')
		else:
			self.htmlUEFailureMsg = 'oaitun_ue1 interface is either NOT mounted or NOT configured'
			self.CreateHtmlTestRow(self.Initialize_OAI_UE_args, 'KO', OAI_UE_PROCESS_NO_TUNNEL_INTERFACE, 'OAI UE')
			logging.error('\033[91mInitialize OAI UE Failed! \033[0m')
			self.AutoTerminateUEandeNB()
			self.CreateHtmlTabFooter(False)
			sys.exit(1)
943

944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030
	def checkDevTTYisUnlocked(self):
		self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
		count = 0
		while count < 5:
			self.command('echo ' + self.ADBPassword + ' | sudo -S lsof | grep ttyUSB0', '\$', 10)
			result = re.search('picocom', str(self.ssh.before))
			if result is None:
				count = 10
			else:
				time.sleep(5)
				count = count + 1
		self.close()

	def InitializeCatM(self):
		if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '':
			Usage()
			sys.exit('Insufficient Parameter')
		self.picocom_closure = True
		self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
		# dummy call to start a sudo session. The picocom command does NOT handle well the `sudo -S`
		self.command('echo ' + self.ADBPassword + ' | sudo -S ls', '\$', 10)
		self.command('sudo picocom --baud 921600 --flow n --databits 8 /dev/ttyUSB0', 'Terminal ready', 10)
		time.sleep(1)
		# Calling twice AT to clear all buffers
		self.command('AT', 'OK|ERROR', 5)
		self.command('AT', 'OK', 5)
		# Disabling the Radio
		self.command('AT+CFUN=0', 'OK', 5)
		logging.debug('\u001B[1m Cellular Functionality disabled\u001B[0m')
		# Checking if auto-attach is enabled
		self.command('AT^AUTOATT?', 'OK', 5)
		result = re.search('AUTOATT: (?P<state>[0-9\-]+)', str(self.ssh.before))
		if result is not None:
			if result.group('state') is not None:
				autoAttachState = int(result.group('state'))
				if autoAttachState is not None:
					if autoAttachState == 0:
						self.command('AT^AUTOATT=1', 'OK', 5)
					logging.debug('\u001B[1m Auto-Attach enabled\u001B[0m')
		else:
			logging.debug('\u001B[1;37;41m Could not check Auto-Attach! \u001B[0m')
		# Force closure of picocom but device might still be locked
		self.close()
		self.picocom_closure = False
		self.CreateHtmlTestRow('N/A', 'OK', ALL_PROCESSES_OK)
		self.checkDevTTYisUnlocked()

	def TerminateCatM(self):
		if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '':
			Usage()
			sys.exit('Insufficient Parameter')
		self.picocom_closure = True
		self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
		# dummy call to start a sudo session. The picocom command does NOT handle well the `sudo -S`
		self.command('echo ' + self.ADBPassword + ' | sudo -S ls', '\$', 10)
		self.command('sudo picocom --baud 921600 --flow n --databits 8 /dev/ttyUSB0', 'Terminal ready', 10)
		time.sleep(1)
		# Calling twice AT to clear all buffers
		self.command('AT', 'OK|ERROR', 5)
		self.command('AT', 'OK', 5)
		# Disabling the Radio
		self.command('AT+CFUN=0', 'OK', 5)
		logging.debug('\u001B[1m Cellular Functionality disabled\u001B[0m')
		self.close()
		self.picocom_closure = False
		self.CreateHtmlTestRow('N/A', 'OK', ALL_PROCESSES_OK)
		self.checkDevTTYisUnlocked()

	def AttachCatM(self):
		if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '':
			Usage()
			sys.exit('Insufficient Parameter')
		self.picocom_closure = True
		self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
		# dummy call to start a sudo session. The picocom command does NOT handle well the `sudo -S`
		self.command('echo ' + self.ADBPassword + ' | sudo -S ls', '\$', 10)
		self.command('sudo picocom --baud 921600 --flow n --databits 8 /dev/ttyUSB0', 'Terminal ready', 10)
		time.sleep(1)
		# Calling twice AT to clear all buffers
		self.command('AT', 'OK|ERROR', 5)
		self.command('AT', 'OK', 5)
		# Enabling the Radio
		self.command('AT+CFUN=1', 'SIMSTORE,READY', 5)
		logging.debug('\u001B[1m Cellular Functionality enabled\u001B[0m')
		time.sleep(4)
		# We should check if we register
		count = 0
1031 1032 1033
		attach_cnt = 0
		attach_status = False
		while count < 5:
1034
			self.command('AT+CEREG?', 'OK', 5)
1035
			result = re.search('CEREG: 2,(?P<state>[0-9\-]+),', str(self.ssh.before))
1036 1037 1038
			if result is not None:
				mDataConnectionState = int(result.group('state'))
				if mDataConnectionState is not None:
1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051
					if mDataConnectionState == 1:
						count = 10
						attach_status = True
						result = re.search('CEREG: 2,1,"(?P<networky>[0-9A-Z]+)","(?P<networkz>[0-9A-Z]+)"', str(self.ssh.before))
						if result is not None:
							networky = result.group('networky')
							networkz = result.group('networkz')
							logging.debug('\u001B[1m CAT-M module attached to eNB (' + str(networky) + '/' + str(networkz) + ')\u001B[0m')
						else:
							logging.debug('\u001B[1m CAT-M module attached to eNB\u001B[0m')
					else:
						logging.debug('+CEREG: 2,' + str(mDataConnectionState))
						attach_cnt = attach_cnt + 1
1052 1053
			else:
				logging.debug(str(self.ssh.before))
1054
				attach_cnt = attach_cnt + 1
1055 1056
			count = count + 1
			time.sleep(1)
1057 1058 1059 1060 1061 1062 1063 1064 1065
		if attach_status:
			self.command('AT+CESQ', 'OK', 5)
			result = re.search('CESQ: 99,99,255,255,(?P<rsrq>[0-9]+),(?P<rsrp>[0-9]+)', str(self.ssh.before))
			if result is not None:
				nRSRQ = int(result.group('rsrq'))
				nRSRP = int(result.group('rsrp'))
				if (nRSRQ is not None) and (nRSRP is not None):
					logging.debug('    RSRQ = ' + str(-20+(nRSRQ/2)) + ' dB')
					logging.debug('    RSRP = ' + str(-140+nRSRP) + ' dBm')
1066 1067
		self.close()
		self.picocom_closure = False
1068
		html_queue = SimpleQueue()
1069
		self.checkDevTTYisUnlocked()
1070
		if attach_status:
1071
			html_cell = '<pre style="background-color:white">CAT-M module\nAttachment Completed in ' + str(attach_cnt+4) + ' seconds'
1072 1073 1074 1075 1076 1077 1078 1079
			if (nRSRQ is not None) and (nRSRP is not None):
				html_cell += '\n   RSRQ = ' + str(-20+(nRSRQ/2)) + ' dB'
				html_cell += '\n   RSRP = ' + str(-140+nRSRP) + ' dBm</pre>'
			else:
				html_cell += '</pre>'
			html_queue.put(html_cell)
			self.CreateHtmlTestRowQueue('N/A', 'OK', 1, html_queue)
		else:
1080
			html_cell = '<pre style="background-color:white">CAT-M module\nAttachment Failed</pre>'
1081 1082 1083 1084 1085 1086 1087
			html_queue.put(html_cell)
			self.CreateHtmlTestRowQueue('N/A', 'KO', 1, html_queue)

	def PingCatM(self):
		if self.EPCIPAddress == '' or self.EPCUserName == '' or self.EPCPassword == '' or self.EPCSourceCodePath == '':
			Usage()
			sys.exit('Insufficient Parameter')
1088 1089 1090
		check_eNB = True
		check_OAI_UE = False
		pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE)
1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114
		if (pStatus < 0):
			self.CreateHtmlTestRow(self.ping_args, 'KO', pStatus)
			self.CreateHtmlTabFooter(False)
			sys.exit(1)
		try:
			statusQueue = SimpleQueue()
			lock = Lock()
			self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
			self.command('cd ' + self.EPCSourceCodePath, '\$', 5)
			self.command('cd scripts', '\$', 5)
			if re.match('OAI', self.EPCType, re.IGNORECASE):
				logging.debug('Using the OAI EPC HSS: not implemented yet')
				self.CreateHtmlTestRow(self.ping_args, 'KO', pStatus)
				self.CreateHtmlTabFooter(False)
				sys.exit(1)
			else:
				self.command('egrep --color=never "Allocated ipv4 addr" /opt/ltebox/var/log/xGwLog.0', '\$', 5)
				result = re.search('Allocated ipv4 addr: (?P<ipaddr>[0-9\.]+) from Pool', str(self.ssh.before))
				if result is not None:
					moduleIPAddr = result.group('ipaddr')
				else:
					return
			ping_time = re.findall("-c (\d+)",str(self.ping_args))
			device_id = 'catm'
1115
			ping_status = self.command('stdbuf -o0 ping ' + self.ping_args + ' ' + str(moduleIPAddr) + ' 2>&1 | stdbuf -o0 tee ping_' + self.testCase_id + '_' + device_id + '.log', '\$', int(ping_time[0])*1.5)
1116 1117 1118 1119
			# TIMEOUT CASE
			if ping_status < 0:
				message = 'Ping with UE (' + str(moduleIPAddr) + ') crashed due to TIMEOUT!'
				logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m')
1120
				self.close()
1121 1122 1123 1124 1125 1126
				self.ping_iperf_wrong_exit(lock, moduleIPAddr, device_id, statusQueue, message)
				return
			result = re.search(', (?P<packetloss>[0-9\.]+)% packet loss, time [0-9\.]+ms', str(self.ssh.before))
			if result is None:
				message = 'Packet Loss Not Found!'
				logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m')
1127
				self.close()
1128 1129 1130 1131 1132 1133
				self.ping_iperf_wrong_exit(lock, moduleIPAddr, device_id, statusQueue, message)
				return
			packetloss = result.group('packetloss')
			if float(packetloss) == 100:
				message = 'Packet Loss is 100%'
				logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m')
1134
				self.close()
1135 1136 1137 1138 1139 1140
				self.ping_iperf_wrong_exit(lock, moduleIPAddr, device_id, statusQueue, message)
				return
			result = re.search('rtt min\/avg\/max\/mdev = (?P<rtt_min>[0-9\.]+)\/(?P<rtt_avg>[0-9\.]+)\/(?P<rtt_max>[0-9\.]+)\/[0-9\.]+ ms', str(self.ssh.before))
			if result is None:
				message = 'Ping RTT_Min RTT_Avg RTT_Max Not Found!'
				logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m')
1141
				self.close()
1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179
				self.ping_iperf_wrong_exit(lock, moduleIPAddr, device_id, statusQueue, message)
				return
			rtt_min = result.group('rtt_min')
			rtt_avg = result.group('rtt_avg')
			rtt_max = result.group('rtt_max')
			pal_msg = 'Packet Loss : ' + packetloss + '%'
			min_msg = 'RTT(Min)    : ' + rtt_min + ' ms'
			avg_msg = 'RTT(Avg)    : ' + rtt_avg + ' ms'
			max_msg = 'RTT(Max)    : ' + rtt_max + ' ms'
			lock.acquire()
			logging.debug('\u001B[1;37;44m ping result (' + moduleIPAddr + ') \u001B[0m')
			logging.debug('\u001B[1;34m    ' + pal_msg + '\u001B[0m')
			logging.debug('\u001B[1;34m    ' + min_msg + '\u001B[0m')
			logging.debug('\u001B[1;34m    ' + avg_msg + '\u001B[0m')
			logging.debug('\u001B[1;34m    ' + max_msg + '\u001B[0m')
			qMsg = pal_msg + '\n' + min_msg + '\n' + avg_msg + '\n' + max_msg
			packetLossOK = True
			if packetloss is not None:
				if float(packetloss) > float(self.ping_packetloss_threshold):
					qMsg += '\nPacket Loss too high'
					logging.debug('\u001B[1;37;41m Packet Loss too high \u001B[0m')
					packetLossOK = False
				elif float(packetloss) > 0:
					qMsg += '\nPacket Loss is not 0%'
					logging.debug('\u001B[1;30;43m Packet Loss is not 0% \u001B[0m')
			lock.release()
			self.close()
			html_cell = '<pre style="background-color:white">CAT-M module\nIP Address  : ' + moduleIPAddr + '\n' + qMsg + '</pre>'
			statusQueue.put(html_cell)
			if (packetLossOK):
				self.CreateHtmlTestRowQueue(self.ping_args, 'OK', 1, statusQueue)
			else:
				self.CreateHtmlTestRowQueue(self.ping_args, 'KO', 1, statusQueue)
				self.AutoTerminateUEandeNB()
				self.CreateHtmlTabFooter(False)
				sys.exit(1)
		except:
			os.kill(os.getppid(),signal.SIGUSR1)
1180

Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
1181
	def AttachUE_common(self, device_id, statusQueue, lock):
1182 1183 1184 1185
		try:
			self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
			self.command('stdbuf -o0 adb -s ' + device_id + ' shell /data/local/tmp/on', '\$', 60)
			time.sleep(2)
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
1186 1187
			max_count = 45
			count = max_count
1188 1189 1190 1191 1192
			while count > 0:
				self.command('stdbuf -o0 adb -s ' + device_id + ' shell dumpsys telephony.registry | grep mDataConnectionState', '\$', 15)
				result = re.search('mDataConnectionState.*=(?P<state>[0-9\-]+)', str(self.ssh.before))
				if result is None:
					logging.debug('\u001B[1;37;41m mDataConnectionState Not Found! \u001B[0m')
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
1193 1194 1195 1196 1197 1198
					lock.acquire()
					statusQueue.put(-1)
					statusQueue.put(device_id)
					statusQueue.put('mDataConnectionState Not Found!')
					lock.release()
					break
1199 1200 1201
				mDataConnectionState = int(result.group('state'))
				if mDataConnectionState == 2:
					logging.debug('\u001B[1mUE (' + device_id + ') Attach Completed\u001B[0m')
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
1202 1203 1204 1205 1206
					lock.acquire()
					statusQueue.put(max_count - count)
					statusQueue.put(device_id)
					statusQueue.put('Attach Completed')
					lock.release()
1207 1208 1209
					break
				count = count - 1
				if count == 15 or count == 30:
Raphael Defosseux's avatar
Raphael Defosseux committed
1210
					logging.debug('\u001B[1;30;43m Retry UE (' + device_id + ') Flight Mode Off \u001B[0m')
1211 1212 1213 1214
					self.command('stdbuf -o0 adb -s ' + device_id + ' shell /data/local/tmp/off', '\$', 60)
					time.sleep(0.5)
					self.command('stdbuf -o0 adb -s ' + device_id + ' shell /data/local/tmp/on', '\$', 60)
					time.sleep(0.5)
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
1215
				logging.debug('\u001B[1mWait UE (' + device_id + ') a second until mDataConnectionState=2 (' + str(max_count-count) + ' times)\u001B[0m')
1216 1217 1218
				time.sleep(1)
			if count == 0:
				logging.debug('\u001B[1;37;41m UE (' + device_id + ') Attach Failed \u001B[0m')
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
1219 1220 1221 1222 1223
				lock.acquire()
				statusQueue.put(-1)
				statusQueue.put(device_id)
				statusQueue.put('Attach Failed')
				lock.release()
1224 1225 1226 1227 1228 1229 1230 1231
			self.close()
		except:
			os.kill(os.getppid(),signal.SIGUSR1)

	def AttachUE(self):
		if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '':
			Usage()
			sys.exit('Insufficient Parameter')
1232 1233 1234
		check_eNB = True
		check_OAI_UE = False
		pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE)
Raphael Defosseux's avatar
Raphael Defosseux committed
1235 1236
		if (pStatus < 0):
			self.CreateHtmlTestRow('N/A', 'KO', pStatus)
1237
			self.AutoTerminateUEandeNB()
1238
			self.CreateHtmlTabFooter(False)
Raphael Defosseux's avatar
Raphael Defosseux committed
1239
			sys.exit(1)
1240
		multi_jobs = []
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
1241 1242
		status_queue = SimpleQueue()
		lock = Lock()
1243
		nb_ue_to_connect = 0
1244
		for device_id in self.UEDevices:
1245
			if (self.nbMaxUEtoAttach == -1) or (nb_ue_to_connect < self.nbMaxUEtoAttach):
1246
				self.UEDevicesStatus[nb_ue_to_connect] = UE_STATUS_ATTACHING
1247
				p = Process(target = self.AttachUE_common, args = (device_id, status_queue, lock,))
1248 1249 1250 1251
				p.daemon = True
				p.start()
				multi_jobs.append(p)
			nb_ue_to_connect = nb_ue_to_connect + 1
1252 1253
		for job in multi_jobs:
			job.join()
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
1254 1255

		if (status_queue.empty()):
1256
			self.CreateHtmlTestRow('N/A', 'KO', ALL_PROCESSES_OK)
1257
			self.CreateHtmlTabFooter(False)
1258
			self.AutoTerminateUEandeNB()
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269
			sys.exit(1)
		else:
			attach_status = True
			html_queue = SimpleQueue()
			while (not status_queue.empty()):
				count = status_queue.get()
				if (count < 0):
					attach_status = False
				device_id = status_queue.get()
				message = status_queue.get()
				if (count < 0):
1270
					html_cell = '<pre style="background-color:white">UE (' + device_id + ')\n' + message + '</pre>'
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
1271
				else:
1272
					html_cell = '<pre style="background-color:white">UE (' + device_id + ')\n' + message + ' in ' + str(count + 2) + ' seconds</pre>'
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
1273 1274
				html_queue.put(html_cell)
			if (attach_status):
1275 1276 1277 1278 1279
				cnt = 0
				while cnt < len(self.UEDevices):
					if self.UEDevicesStatus[cnt] == UE_STATUS_ATTACHING:
						self.UEDevicesStatus[cnt] = UE_STATUS_ATTACHED
					cnt += 1
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
1280
				self.CreateHtmlTestRowQueue('N/A', 'OK', len(self.UEDevices), html_queue)
1281 1282 1283 1284
				result = re.search('T_stdout', str(self.Initialize_eNB_args))
				if result is not None:
					logging.debug('Waiting 5 seconds to fill up record file')
					time.sleep(5)
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
1285 1286
			else:
				self.CreateHtmlTestRowQueue('N/A', 'KO', len(self.UEDevices), html_queue)
1287
				self.AutoTerminateUEandeNB()
1288
				self.CreateHtmlTabFooter(False)
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
1289
				sys.exit(1)
1290 1291 1292 1293

	def DetachUE_common(self, device_id):
		try:
			self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
1294
			self.command('stdbuf -o0 adb -s ' + device_id + ' shell /data/local/tmp/off', '\$', 60)
1295 1296 1297 1298 1299 1300 1301 1302 1303
			logging.debug('\u001B[1mUE (' + device_id + ') Detach Completed\u001B[0m')
			self.close()
		except:
			os.kill(os.getppid(),signal.SIGUSR1)

	def DetachUE(self):
		if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '':
			Usage()
			sys.exit('Insufficient Parameter')
1304
		check_eNB = True
Raphael Defosseux's avatar
Raphael Defosseux committed
1305
		check_OAI_UE = False
1306
		pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE)
Raphael Defosseux's avatar
Raphael Defosseux committed
1307 1308
		if (pStatus < 0):
			self.CreateHtmlTestRow('N/A', 'KO', pStatus)
1309
			self.AutoTerminateUEandeNB()
1310
			self.CreateHtmlTabFooter(False)
Raphael Defosseux's avatar
Raphael Defosseux committed
1311
			sys.exit(1)
1312
		multi_jobs = []
1313
		cnt = 0
1314
		for device_id in self.UEDevices:
1315
			self.UEDevicesStatus[cnt] = UE_STATUS_DETACHING
1316
			p = Process(target = self.DetachUE_common, args = (device_id,))
1317 1318 1319
			p.daemon = True
			p.start()
			multi_jobs.append(p)
1320
			cnt += 1
1321 1322
		for job in multi_jobs:
			job.join()
1323
		self.CreateHtmlTestRow('N/A', 'OK', ALL_PROCESSES_OK)
1324 1325 1326 1327
		result = re.search('T_stdout', str(self.Initialize_eNB_args))
		if result is not None:
			logging.debug('Waiting 5 seconds to fill up record file')
			time.sleep(5)
1328 1329 1330 1331
		cnt = 0
		while cnt < len(self.UEDevices):
			self.UEDevicesStatus[cnt] = UE_STATUS_DETACHED
			cnt += 1
1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375

	def RebootUE_common(self, device_id):
		try:
			self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
			previousmDataConnectionStates = []
			# Save mDataConnectionState
			self.command('stdbuf -o0 adb -s ' + device_id + ' shell dumpsys telephony.registry | grep mDataConnectionState', '\$', 15)
			self.command('stdbuf -o0 adb -s ' + device_id + ' shell dumpsys telephony.registry | grep mDataConnectionState', '\$', 15)
			result = re.search('mDataConnectionState.*=(?P<state>[0-9\-]+)', str(self.ssh.before))
			if result is None:
				logging.debug('\u001B[1;37;41m mDataConnectionState Not Found! \u001B[0m')
				sys.exit(1)
			previousmDataConnectionStates.append(int(result.group('state')))
			# Reboot UE
			self.command('stdbuf -o0 adb -s ' + device_id + ' shell reboot', '\$', 10)
			time.sleep(60)
			previousmDataConnectionState = previousmDataConnectionStates.pop(0)
			count = 180
			while count > 0:
				count = count - 1
				self.command('stdbuf -o0 adb -s ' + device_id + ' shell dumpsys telephony.registry | grep mDataConnectionState', '\$', 15)
				result = re.search('mDataConnectionState.*=(?P<state>[0-9\-]+)', str(self.ssh.before))
				if result is None:
					mDataConnectionState = None
				else:
					mDataConnectionState = int(result.group('state'))
					logging.debug('mDataConnectionState = ' + result.group('state'))
				if mDataConnectionState is None or (previousmDataConnectionState == 2 and mDataConnectionState != 2):
					logging.debug('\u001B[1mWait UE (' + device_id + ') a second until reboot completion (' + str(180-count) + ' times)\u001B[0m')
					time.sleep(1)
				else:
					logging.debug('\u001B[1mUE (' + device_id + ') Reboot Completed\u001B[0m')
					break
			if count == 0:
				logging.debug('\u001B[1;37;41m UE (' + device_id + ') Reboot Failed \u001B[0m')
				sys.exit(1)
			self.close()
		except:
			os.kill(os.getppid(),signal.SIGUSR1)

	def RebootUE(self):
		if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '':
			Usage()
			sys.exit('Insufficient Parameter')
1376
		check_eNB = True
Raphael Defosseux's avatar
Raphael Defosseux committed
1377
		check_OAI_UE = False
1378
		pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE)
Raphael Defosseux's avatar
Raphael Defosseux committed
1379 1380
		if (pStatus < 0):
			self.CreateHtmlTestRow('N/A', 'KO', pStatus)
1381
			self.CreateHtmlTabFooter(False)
Raphael Defosseux's avatar
Raphael Defosseux committed
1382
			sys.exit(1)
1383 1384
		multi_jobs = []
		for device_id in self.UEDevices:
1385
			p = Process(target = self.RebootUE_common, args = (device_id,))
1386 1387 1388 1389 1390
			p.daemon = True
			p.start()
			multi_jobs.append(p)
		for job in multi_jobs:
			job.join()
1391
		self.CreateHtmlTestRow('N/A', 'OK', ALL_PROCESSES_OK)
1392

1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440
	def DataDisableUE_common(self, device_id):
		try:
			self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
			# enable data service
			self.command('stdbuf -o0 adb -s ' + device_id + ' shell svc data disable', '\$', 60)
			logging.debug('\u001B[1mUE (' + device_id + ') Disabled Data Service\u001B[0m')
			self.close()
		except:
			os.kill(os.getppid(),signal.SIGUSR1)

	def DataDisableUE(self):
		if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '':
			Usage()
			sys.exit('Insufficient Parameter')
		multi_jobs = []
		for device_id in self.UEDevices:
			p = Process(target = self.DataDisableUE_common, args = (device_id,))
			p.daemon = True
			p.start()
			multi_jobs.append(p)
		for job in multi_jobs:
			job.join()
		self.CreateHtmlTestRow('N/A', 'OK', ALL_PROCESSES_OK)

	def DataEnableUE_common(self, device_id):
		try:
			self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
			# enable data service
			self.command('stdbuf -o0 adb -s ' + device_id + ' shell svc data enable', '\$', 60)
			logging.debug('\u001B[1mUE (' + device_id + ') Enabled Data Service\u001B[0m')
			self.close()
		except:
			os.kill(os.getppid(),signal.SIGUSR1)

	def DataEnableUE(self):
		if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '':
			Usage()
			sys.exit('Insufficient Parameter')
		multi_jobs = []
		for device_id in self.UEDevices:
			p = Process(target = self.DataEnableUE_common, args = (device_id,))
			p.daemon = True
			p.start()
			multi_jobs.append(p)
		for job in multi_jobs:
			job.join()
		self.CreateHtmlTestRow('N/A', 'OK', ALL_PROCESSES_OK)

1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451
	def GetAllUEDevices(self, terminate_ue_flag):
		if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '':
			Usage()
			sys.exit('Insufficient Parameter')
		self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
		self.command('adb devices', '\$', 15)
		self.UEDevices = re.findall("\\\\r\\\\n([A-Za-z0-9]+)\\\\tdevice",str(self.ssh.before))
		if terminate_ue_flag == False:
			if len(self.UEDevices) == 0:
				logging.debug('\u001B[1;37;41m UE Not Found! \u001B[0m')
				sys.exit(1)
1452 1453 1454 1455 1456
		if len(self.UEDevicesStatus) == 0:
			cnt = 0
			while cnt < len(self.UEDevices):
				self.UEDevicesStatus.append(UE_STATUS_DETACHED)
				cnt += 1
1457 1458
		self.close()

1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471
	def GetAllCatMDevices(self, terminate_ue_flag):
		if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '':
			Usage()
			sys.exit('Insufficient Parameter')
		self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
		self.command('lsusb | egrep "Future Technology Devices International, Ltd FT2232C" | sed -e "s#:.*##" -e "s# #_#g"', '\$', 15)
		self.CatMDevices = re.findall("\\\\r\\\\n([A-Za-z0-9_]+)",str(self.ssh.before))
		if terminate_ue_flag == False:
			if len(self.CatMDevices) == 0:
				logging.debug('\u001B[1;37;41m CAT-M UE Not Found! \u001B[0m')
				sys.exit(1)
		self.close()

1472 1473 1474
	def CheckUEStatus_common(self, lock, device_id, statusQueue):
		try:
			self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503
			self.command('stdbuf -o0 adb -s ' + device_id + ' shell dumpsys telephony.registry', '\$', 15)
			result = re.search('mServiceState=(?P<serviceState>[0-9]+)', str(self.ssh.before))
			serviceState = 'Service State: UNKNOWN'
			if result is not None:
				lServiceState = int(result.group('serviceState'))
				if lServiceState == 3:
					serviceState = 'Service State: RADIO_POWERED_OFF'
				if lServiceState == 1:
					serviceState = 'Service State: OUT_OF_SERVICE'
				if lServiceState == 0:
					serviceState = 'Service State: IN_SERVICE'
				if lServiceState == 2:
					serviceState = 'Service State: EMERGENCY_ONLY'
			result = re.search('mDataConnectionState=(?P<dataConnectionState>[0-9]+)', str(self.ssh.before))
			dataConnectionState = 'Data State:    UNKNOWN'
			if result is not None:
				lDataConnectionState = int(result.group('dataConnectionState'))
				if lDataConnectionState == 0:
					dataConnectionState = 'Data State:    DISCONNECTED'
				if lDataConnectionState == 1:
					dataConnectionState = 'Data State:    CONNECTING'
				if lDataConnectionState == 2:
					dataConnectionState = 'Data State:    CONNECTED'
				if lDataConnectionState == 3:
					dataConnectionState = 'Data State:    SUSPENDED'
			result = re.search('mDataConnectionReason=(?P<dataConnectionReason>[0-9a-zA-Z_]+)', str(self.ssh.before))
			dataConnectionReason = 'Data Reason:   UNKNOWN'
			if result is not None:
				dataConnectionReason = 'Data Reason:   ' + result.group('dataConnectionReason')
1504 1505
			lock.acquire()
			logging.debug('\u001B[1;37;44m Status Check (' + str(device_id) + ') \u001B[0m')
1506 1507 1508
			logging.debug('\u001B[1;34m    ' + serviceState + '\u001B[0m')
			logging.debug('\u001B[1;34m    ' + dataConnectionState + '\u001B[0m')
			logging.debug('\u001B[1;34m    ' + dataConnectionReason + '\u001B[0m')
1509 1510
			statusQueue.put(0)
			statusQueue.put(device_id)
1511 1512
			qMsg = serviceState + '\n' + dataConnectionState + '\n' + dataConnectionReason
			statusQueue.put(qMsg)
1513 1514 1515 1516 1517 1518 1519 1520 1521
			lock.release()
			self.close()
		except:
			os.kill(os.getppid(),signal.SIGUSR1)

	def CheckStatusUE(self):
		if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '':
			Usage()
			sys.exit('Insufficient Parameter')
1522 1523 1524
		check_eNB = True
		check_OAI_UE = False
		pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE)
1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538
		if (pStatus < 0):
			self.CreateHtmlTestRow('N/A', 'KO', pStatus)
			self.CreateHtmlTabFooter(False)
			sys.exit(1)
		multi_jobs = []
		lock = Lock()
		status_queue = SimpleQueue()
		for device_id in self.UEDevices:
			p = Process(target = self.CheckUEStatus_common, args = (lock,device_id,status_queue,))
			p.daemon = True
			p.start()
			multi_jobs.append(p)
		for job in multi_jobs:
			job.join()
1539 1540 1541 1542 1543 1544 1545 1546 1547 1548
		if self.flexranCtrlInstalled and self.flexranCtrlStarted:
			self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
			self.command('cd /opt/flexran_rtc', '\$', 5)
			self.command('curl http://localhost:9999/stats | jq \'.\' > log/check_status_' + self.testCase_id + '.log 2>&1', '\$', 5)
			self.command('cat log/check_status_' + self.testCase_id + '.log | jq \'.eNB_config[0].UE\' | grep -c rnti | sed -e "s#^#Nb Connected UE = #"', '\$', 5)
			result = re.search('Nb Connected UE = (?P<nb_ues>[0-9]+)', str(self.ssh.before))
			passStatus = True
			if result is not None:
				nb_ues = int(result.group('nb_ues'))
				htmlOptions = 'Nb Connected UE(s) to eNB = ' + str(nb_ues)
1549
				logging.debug('\u001B[1;37;44m ' + htmlOptions + ' \u001B[0m')
1550 1551 1552 1553 1554 1555 1556
				if self.expectedNbOfConnectedUEs > -1:
					if nb_ues != self.expectedNbOfConnectedUEs:
						passStatus = False
			else:
				htmlOptions = 'N/A'
			self.close()
		else:
1557
			passStatus = True
1558
			htmlOptions = 'N/A'
1559 1560

		if (status_queue.empty()):
1561
			self.CreateHtmlTestRow(htmlOptions, 'KO', ALL_PROCESSES_OK)
1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575
			self.AutoTerminateUEandeNB()
			self.CreateHtmlTabFooter(False)
			sys.exit(1)
		else:
			check_status = True
			html_queue = SimpleQueue()
			while (not status_queue.empty()):
				count = status_queue.get()
				if (count < 0):
					check_status = False
				device_id = status_queue.get()
				message = status_queue.get()
				html_cell = '<pre style="background-color:white">UE (' + device_id + ')\n' + message + '</pre>'
				html_queue.put(html_cell)
1576 1577
			if check_status and passStatus:
				self.CreateHtmlTestRowQueue(htmlOptions, 'OK', len(self.UEDevices), html_queue)
1578
			else:
1579
				self.CreateHtmlTestRowQueue(htmlOptions, 'KO', len(self.UEDevices), html_queue)
1580 1581 1582 1583
				self.AutoTerminateUEandeNB()
				self.CreateHtmlTabFooter(False)
				sys.exit(1)

1584 1585 1586 1587
	def GetAllUEIPAddresses(self):
		if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '':
			Usage()
			sys.exit('Insufficient Parameter')
1588
		ue_ip_status = 0
1589
		self.UEIPAddresses = []
1590 1591 1592 1593 1594 1595 1596
		if (len(self.UEDevices) == 1) and (self.UEDevices[0] == 'OAI-UE'):
			if self.UEIPAddress == '' or self.UEUserName == '' or self.UEPassword == '' or self.UESourceCodePath == '':
				Usage()
				sys.exit('Insufficient Parameter')
			self.open(self.UEIPAddress, self.UEUserName, self.UEPassword)
			self.command('ifconfig oaitun_ue1', '\$', 4)
			result = re.search('inet addr:(?P<ueipaddress>[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)', str(self.ssh.before))
1597 1598 1599 1600 1601 1602 1603
			if result is not None:
				UE_IPAddress = result.group('ueipaddress')
				logging.debug('\u001B[1mUE (' + self.UEDevices[0] + ') IP Address is ' + UE_IPAddress + '\u001B[0m')
				self.UEIPAddresses.append(UE_IPAddress)
			else:
				logging.debug('\u001B[1;37;41m UE IP Address Not Found! \u001B[0m')
				ue_ip_status -= 1
1604 1605
			self.close()
			return ue_ip_status
1606
		self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
1607
		idx = 0
1608
		for device_id in self.UEDevices:
1609 1610 1611
			if self.UEDevicesStatus[idx] != UE_STATUS_ATTACHED:
				idx += 1
				continue
1612 1613 1614 1615 1616 1617
			count = 0
			while count < 4:
				self.command('stdbuf -o0 adb -s ' + device_id + ' shell ip addr show | grep rmnet', '\$', 15)
				result = re.search('inet (?P<ueipaddress>[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)\/[0-9]+[0-9a-zA-Z\.\s]+', str(self.ssh.before))
				if result is None:
					logging.debug('\u001B[1;37;41m UE IP Address Not Found! \u001B[0m')
1618
					time.sleep(1)
1619 1620 1621 1622 1623 1624
					count += 1
				else:
					count = 10
			if count < 9:
				ue_ip_status -= 1
				continue
1625 1626 1627 1628
			UE_IPAddress = result.group('ueipaddress')
			logging.debug('\u001B[1mUE (' + device_id + ') IP Address is ' + UE_IPAddress + '\u001B[0m')
			for ueipaddress in self.UEIPAddresses:
				if ueipaddress == UE_IPAddress:
1629
					logging.debug('\u001B[1mUE (' + device_id + ') IP Address ' + UE_IPAddress + ': has already been allocated to another device !' + '\u001B[0m')
1630 1631
					ue_ip_status -= 1
					continue
1632
			self.UEIPAddresses.append(UE_IPAddress)
1633
			idx += 1
1634
		self.close()
1635
		return ue_ip_status
1636

Raphael Defosseux's avatar
Raphael Defosseux committed
1637 1638 1639 1640 1641 1642 1643 1644 1645
	def ping_iperf_wrong_exit(self, lock, UE_IPAddress, device_id, statusQueue, message):
		lock.acquire()
		statusQueue.put(-1)
		statusQueue.put(device_id)
		statusQueue.put(UE_IPAddress)
		statusQueue.put(message)
		lock.release()

	def Ping_common(self, lock, UE_IPAddress, device_id, statusQueue):
1646 1647 1648 1649 1650
		try:
			self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
			self.command('cd ' + self.EPCSourceCodePath, '\$', 5)
			self.command('cd scripts', '\$', 5)
			ping_time = re.findall("-c (\d+)",str(self.ping_args))
1651
			ping_status = self.command('stdbuf -o0 ping ' + self.ping_args + ' ' + UE_IPAddress + ' 2>&1 | stdbuf -o0 tee ping_' + self.testCase_id + '_' + device_id + '.log', '\$', int(ping_time[0])*1.5)
Raphael Defosseux's avatar
Raphael Defosseux committed
1652 1653 1654 1655
			# TIMEOUT CASE
			if ping_status < 0:
				message = 'Ping with UE (' + str(UE_IPAddress) + ') crashed due to TIMEOUT!'
				logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m')
1656
				self.close()
Raphael Defosseux's avatar
Raphael Defosseux committed
1657 1658
				self.ping_iperf_wrong_exit(lock, UE_IPAddress, device_id, statusQueue, message)
				return
1659 1660
			result = re.search(', (?P<packetloss>[0-9\.]+)% packet loss, time [0-9\.]+ms', str(self.ssh.before))
			if result is None:
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
1661 1662
				message = 'Packet Loss Not Found!'
				logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m')
1663
				self.close()
Raphael Defosseux's avatar
Raphael Defosseux committed
1664
				self.ping_iperf_wrong_exit(lock, UE_IPAddress, device_id, statusQueue, message)
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
1665
				return
1666 1667
			packetloss = result.group('packetloss')
			if float(packetloss) == 100:
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
1668 1669
				message = 'Packet Loss is 100%'
				logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m')
1670
				self.close()
Raphael Defosseux's avatar
Raphael Defosseux committed
1671
				self.ping_iperf_wrong_exit(lock, UE_IPAddress, device_id, statusQueue, message)
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
1672
				return
1673 1674
			result = re.search('rtt min\/avg\/max\/mdev = (?P<rtt_min>[0-9\.]+)\/(?P<rtt_avg>[0-9\.]+)\/(?P<rtt_max>[0-9\.]+)\/[0-9\.]+ ms', str(self.ssh.before))
			if result is None:
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
1675 1676
				message = 'Ping RTT_Min RTT_Avg RTT_Max Not Found!'
				logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m')
1677
				self.close()
Raphael Defosseux's avatar
Raphael Defosseux committed
1678
				self.ping_iperf_wrong_exit(lock, UE_IPAddress, device_id, statusQueue, message)
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
1679
				return
1680 1681 1682
			rtt_min = result.group('rtt_min')
			rtt_avg = result.group('rtt_avg')
			rtt_max = result.group('rtt_max')
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
1683 1684 1685 1686
			pal_msg = 'Packet Loss : ' + packetloss + '%'
			min_msg = 'RTT(Min)    : ' + rtt_min + ' ms'
			avg_msg = 'RTT(Avg)    : ' + rtt_avg + ' ms'
			max_msg = 'RTT(Max)    : ' + rtt_max + ' ms'
1687 1688
			lock.acquire()
			logging.debug('\u001B[1;37;44m ping result (' + UE_IPAddress + ') \u001B[0m')
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
1689 1690 1691 1692 1693 1694
			logging.debug('\u001B[1;34m    ' + pal_msg + '\u001B[0m')
			logging.debug('\u001B[1;34m    ' + min_msg + '\u001B[0m')
			logging.debug('\u001B[1;34m    ' + avg_msg + '\u001B[0m')
			logging.debug('\u001B[1;34m    ' + max_msg + '\u001B[0m')
			qMsg = pal_msg + '\n' + min_msg + '\n' + avg_msg + '\n' + max_msg
			packetLossOK = True
1695 1696
			if packetloss is not None:
				if float(packetloss) > float(self.ping_packetloss_threshold):
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
1697
					qMsg += '\nPacket Loss too high'
1698
					logging.debug('\u001B[1;37;41m Packet Loss too high \u001B[0m')
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
1699
					packetLossOK = False
1700
				elif float(packetloss) > 0:
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
1701
					qMsg += '\nPacket Loss is not 0%'
Raphael Defosseux's avatar
Raphael Defosseux committed
1702
					logging.debug('\u001B[1;30;43m Packet Loss is not 0% \u001B[0m')
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
1703 1704 1705 1706 1707 1708 1709 1710
			if (packetLossOK):
				statusQueue.put(0)
			else:
				statusQueue.put(-1)
			statusQueue.put(device_id)
			statusQueue.put(UE_IPAddress)
			statusQueue.put(qMsg)
			lock.release()
1711 1712 1713 1714
			self.close()
		except:
			os.kill(os.getppid(),signal.SIGUSR1)

1715 1716 1717 1718 1719 1720
	def PingNoS1_wrong_exit(self, qMsg):
		html_queue = SimpleQueue()
		html_cell = '<pre style="background-color:white">OAI UE ping result\n' + qMsg + '</pre>'
		html_queue.put(html_cell)
		self.CreateHtmlTestRowQueue(self.ping_args, 'KO', len(self.UEDevices), html_queue)

1721
	def PingNoS1(self):
1722 1723 1724 1725 1726 1727 1728 1729
		check_eNB = True
		check_OAI_UE = True
		pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE)
		if (pStatus < 0):
			self.CreateHtmlTestRow(self.ping_args, 'KO', pStatus)
			self.AutoTerminateUEandeNB()
			self.CreateHtmlTabFooter(False)
			sys.exit(1)
1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747
		ping_from_eNB = re.search('oaitun_enb1', str(self.ping_args))
		if ping_from_eNB is not None:
			if self.eNBIPAddress == '' or self.eNBUserName == '' or self.eNBPassword == '':
				Usage()
				sys.exit('Insufficient Parameter')
		else:
			if self.UEIPAddress == '' or self.UEUserName == '' or self.UEPassword == '':
				Usage()
				sys.exit('Insufficient Parameter')
		try:
			if ping_from_eNB is not None:
				self.open(self.eNBIPAddress, self.eNBUserName, self.eNBPassword)
				self.command('cd ' + self.eNBSourceCodePath + '/cmake_targets/', '\$', 5)
			else:
				self.open(self.UEIPAddress, self.UEUserName, self.UEPassword)
				self.command('cd ' + self.UESourceCodePath + '/cmake_targets/', '\$', 5)
			self.command('cd cmake_targets', '\$', 5)
			ping_time = re.findall("-c (\d+)",str(self.ping_args))
1748
			ping_status = self.command('stdbuf -o0 ping ' + self.ping_args + ' 2>&1 | stdbuf -o0 tee ping_' + self.testCase_id + '.log', '\$', int(ping_time[0])*1.5)
1749 1750 1751 1752
			# TIMEOUT CASE
			if ping_status < 0:
				message = 'Ping with OAI UE crashed due to TIMEOUT!'
				logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m')
1753
				self.PingNoS1_wrong_exit(message)
1754 1755 1756 1757 1758
				return
			result = re.search(', (?P<packetloss>[0-9\.]+)% packet loss, time [0-9\.]+ms', str(self.ssh.before))
			if result is None:
				message = 'Packet Loss Not Found!'
				logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m')
1759
				self.PingNoS1_wrong_exit(message)
1760 1761 1762 1763 1764
				return
			packetloss = result.group('packetloss')
			if float(packetloss) == 100:
				message = 'Packet Loss is 100%'
				logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m')
1765
				self.PingNoS1_wrong_exit(message)
1766 1767 1768 1769 1770
				return
			result = re.search('rtt min\/avg\/max\/mdev = (?P<rtt_min>[0-9\.]+)\/(?P<rtt_avg>[0-9\.]+)\/(?P<rtt_max>[0-9\.]+)\/[0-9\.]+ ms', str(self.ssh.before))
			if result is None:
				message = 'Ping RTT_Min RTT_Avg RTT_Max Not Found!'
				logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m')
1771
				self.PingNoS1_wrong_exit(message)
1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803
				return
			rtt_min = result.group('rtt_min')
			rtt_avg = result.group('rtt_avg')
			rtt_max = result.group('rtt_max')
			pal_msg = 'Packet Loss : ' + packetloss + '%'
			min_msg = 'RTT(Min)    : ' + rtt_min + ' ms'
			avg_msg = 'RTT(Avg)    : ' + rtt_avg + ' ms'
			max_msg = 'RTT(Max)    : ' + rtt_max + ' ms'
			logging.debug('\u001B[1;37;44m OAI UE ping result \u001B[0m')
			logging.debug('\u001B[1;34m    ' + pal_msg + '\u001B[0m')
			logging.debug('\u001B[1;34m    ' + min_msg + '\u001B[0m')
			logging.debug('\u001B[1;34m    ' + avg_msg + '\u001B[0m')
			logging.debug('\u001B[1;34m    ' + max_msg + '\u001B[0m')
			qMsg = pal_msg + '\n' + min_msg + '\n' + avg_msg + '\n' + max_msg
			packetLossOK = True
			if packetloss is not None:
				if float(packetloss) > float(self.ping_packetloss_threshold):
					qMsg += '\nPacket Loss too high'
					logging.debug('\u001B[1;37;41m Packet Loss too high \u001B[0m')
					packetLossOK = False
				elif float(packetloss) > 0:
					qMsg += '\nPacket Loss is not 0%'
					logging.debug('\u001B[1;30;43m Packet Loss is not 0% \u001B[0m')
			self.close()
			html_queue = SimpleQueue()
			ip_addr = 'TBD'
			html_cell = '<pre style="background-color:white">OAI UE ping result\n' + qMsg + '</pre>'
			html_queue.put(html_cell)
			if packetLossOK:
				self.CreateHtmlTestRowQueue(self.ping_args, 'OK', len(self.UEDevices), html_queue)
			else:
				self.CreateHtmlTestRowQueue(self.ping_args, 'KO', len(self.UEDevices), html_queue)
1804 1805 1806 1807 1808 1809 1810 1811

			# copying on the EPC server for logCollection
			if ping_from_eNB is not None:
				copyin_res = self.copyin(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, self.eNBSourceCodePath + '/cmake_targets/ping_' + self.testCase_id + '.log', '.')
			else:
				copyin_res = self.copyin(self.UEIPAddress, self.UEUserName, self.UEPassword, self.UESourceCodePath + '/cmake_targets/ping_' + self.testCase_id + '.log', '.')
			if (copyin_res == 0):
				self.copyout(self.EPCIPAddress, self.EPCUserName, self.EPCPassword, 'ping_' + self.testCase_id + '.log', self.EPCSourceCodePath + '/scripts')
1812 1813 1814
		except:
			os.kill(os.getppid(),signal.SIGUSR1)

1815
	def Ping(self):
1816 1817 1818 1819
		result = re.search('noS1', str(self.Initialize_eNB_args))
		if result is not None:
			self.PingNoS1()
			return
1820 1821 1822
		if self.EPCIPAddress == '' or self.EPCUserName == '' or self.EPCPassword == '' or self.EPCSourceCodePath == '':
			Usage()
			sys.exit('Insufficient Parameter')
1823
		check_eNB = True
1824 1825 1826 1827
		if (len(self.UEDevices) == 1) and (self.UEDevices[0] == 'OAI-UE'):
			check_OAI_UE = True
		else:
			check_OAI_UE = False
1828
		pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE)
Raphael Defosseux's avatar
Raphael Defosseux committed
1829 1830
		if (pStatus < 0):
			self.CreateHtmlTestRow(self.ping_args, 'KO', pStatus)
1831
			self.AutoTerminateUEandeNB()
1832
			self.CreateHtmlTabFooter(False)
Raphael Defosseux's avatar
Raphael Defosseux committed
1833
			sys.exit(1)
1834 1835 1836
		ueIpStatus = self.GetAllUEIPAddresses()
		if (ueIpStatus < 0):
			self.CreateHtmlTestRow(self.ping_args, 'KO', UE_IP_ADDRESS_ISSUE)
1837
			self.AutoTerminateUEandeNB()
1838
			self.CreateHtmlTabFooter(False)
1839
			sys.exit(1)
1840 1841 1842
		multi_jobs = []
		i = 0
		lock = Lock()
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
1843
		status_queue = SimpleQueue()
1844 1845
		for UE_IPAddress in self.UEIPAddresses:
			device_id = self.UEDevices[i]
1846
			p = Process(target = self.Ping_common, args = (lock,UE_IPAddress,device_id,status_queue,))
1847 1848 1849 1850 1851 1852
			p.daemon = True
			p.start()
			multi_jobs.append(p)
			i = i + 1
		for job in multi_jobs:
			job.join()
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
1853 1854

		if (status_queue.empty()):
1855
			self.CreateHtmlTestRow(self.ping_args, 'KO', ALL_PROCESSES_OK)
1856
			self.AutoTerminateUEandeNB()
1857
			self.CreateHtmlTabFooter(False)
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868
			sys.exit(1)
		else:
			ping_status = True
			html_queue = SimpleQueue()
			while (not status_queue.empty()):
				count = status_queue.get()
				if (count < 0):
					ping_status = False
				device_id = status_queue.get()
				ip_addr = status_queue.get()
				message = status_queue.get()
1869
				html_cell = '<pre style="background-color:white">UE (' + device_id + ')\nIP Address  : ' + ip_addr + '\n' + message + '</pre>'
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
1870 1871 1872 1873 1874
				html_queue.put(html_cell)
			if (ping_status):
				self.CreateHtmlTestRowQueue(self.ping_args, 'OK', len(self.UEDevices), html_queue)
			else:
				self.CreateHtmlTestRowQueue(self.ping_args, 'KO', len(self.UEDevices), html_queue)
1875
				self.AutoTerminateUEandeNB()
1876
				self.CreateHtmlTabFooter(False)
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
1877
				sys.exit(1)
1878

1879 1880 1881 1882 1883 1884 1885
	def Iperf_ComputeTime(self):
		result = re.search('-t (?P<iperf_time>\d+)', str(self.iperf_args))
		if result is None:
			logging.debug('\u001B[1;37;41m Iperf time Not Found! \u001B[0m')
			sys.exit(1)
		return result.group('iperf_time')

1886
	def Iperf_ComputeModifiedBW(self, idx, ue_num):
1887 1888 1889 1890 1891
		result = re.search('-b (?P<iperf_bandwidth>[0-9\.]+)[KMG]', str(self.iperf_args))
		if result is None:
			logging.debug('\u001B[1;37;41m Iperf bandwidth Not Found! \u001B[0m')
			sys.exit(1)
		iperf_bandwidth = result.group('iperf_bandwidth')
1892
		if self.iperf_profile == 'balanced':
1893
			iperf_bandwidth_new = float(iperf_bandwidth)/ue_num
1894
		if self.iperf_profile == 'single-ue':
1895
			iperf_bandwidth_new = float(iperf_bandwidth)
1896
		if self.iperf_profile == 'unbalanced':
1897 1898 1899 1900 1901 1902
			# residual is 2% of max bw
			residualBW = float(iperf_bandwidth) / 50
			if idx == 0:
				iperf_bandwidth_new = float(iperf_bandwidth) - ((ue_num - 1) * residualBW)
			else:
				iperf_bandwidth_new = residualBW
1903
		iperf_bandwidth_str = '-b ' + iperf_bandwidth
1904
		iperf_bandwidth_str_new = '-b ' + ('%.2f' % iperf_bandwidth_new)
1905 1906 1907 1908 1909 1910
		result = re.sub(iperf_bandwidth_str, iperf_bandwidth_str_new, str(self.iperf_args))
		if result is None:
			logging.debug('\u001B[1;37;41m Calculate Iperf bandwidth Failed! \u001B[0m')
			sys.exit(1)
		return result

1911
	def Iperf_analyzeV2TCPOutput(self, lock, UE_IPAddress, device_id, statusQueue, iperf_real_options):
1912
		self.command('awk -f /tmp/tcp_iperf_stats.awk /tmp/CI-eNB/scripts/iperf_' + self.testCase_id + '_' + device_id + '.log', '\$', 5)
1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936
		result = re.search('Avg Bitrate : (?P<average>[0-9\.]+ Mbits\/sec) Max Bitrate : (?P<maximum>[0-9\.]+ Mbits\/sec) Min Bitrate : (?P<minimum>[0-9\.]+ Mbits\/sec)', str(self.ssh.before))
		if result is not None:
			avgbitrate = result.group('average')
			maxbitrate = result.group('maximum')
			minbitrate = result.group('minimum')
			lock.acquire()
			logging.debug('\u001B[1;37;44m TCP iperf result (' + UE_IPAddress + ') \u001B[0m')
			msg = 'TCP Stats   :\n'
			if avgbitrate is not None:
				logging.debug('\u001B[1;34m    Avg Bitrate : ' + avgbitrate + '\u001B[0m')
				msg += 'Avg Bitrate : ' + avgbitrate + '\n'
			if maxbitrate is not None:
				logging.debug('\u001B[1;34m    Max Bitrate : ' + maxbitrate + '\u001B[0m')
				msg += 'Max Bitrate : ' + maxbitrate + '\n'
			if minbitrate is not None:
				logging.debug('\u001B[1;34m    Min Bitrate : ' + minbitrate + '\u001B[0m')
				msg += 'Min Bitrate : ' + minbitrate + '\n'
			statusQueue.put(0)
			statusQueue.put(device_id)
			statusQueue.put(UE_IPAddress)
			statusQueue.put(msg)
			lock.release()
		return 0

1937
	def Iperf_analyzeV2Output(self, lock, UE_IPAddress, device_id, statusQueue, iperf_real_options):
1938 1939 1940 1941
		result = re.search('-u', str(iperf_real_options))
		if result is None:
			return self.Iperf_analyzeV2TCPOutput(lock, UE_IPAddress, device_id, statusQueue, iperf_real_options)

1942 1943 1944 1945 1946 1947 1948
		result = re.search('Server Report:', str(self.ssh.before))
		if result is None:
			result = re.search('read failed: Connection refused', str(self.ssh.before))
			if result is not None:
				logging.debug('\u001B[1;37;41m Could not connect to iperf server! \u001B[0m')
			else:
				logging.debug('\u001B[1;37;41m Server Report and Connection refused Not Found! \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
1949
			return -1
1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967
		# Computing the requested bandwidth in float
		result = re.search('-b (?P<iperf_bandwidth>[0-9\.]+)[KMG]', str(iperf_real_options))
		if result is not None:
			req_bandwidth = result.group('iperf_bandwidth')
			req_bw = float(req_bandwidth)
			result = re.search('-b [0-9\.]+K', str(iperf_real_options))
			if result is not None:
				req_bandwidth = '%.1f Kbits/sec' % req_bw
				req_bw = req_bw * 1000
			result = re.search('-b [0-9\.]+M', str(iperf_real_options))
			if result is not None:
				req_bandwidth = '%.1f Mbits/sec' % req_bw
				req_bw = req_bw * 1000000
			result = re.search('-b [0-9\.]+G', str(iperf_real_options))
			if result is not None:
				req_bandwidth = '%.1f Gbits/sec' % req_bw
				req_bw = req_bw * 1000000000

1968
		result = re.search('Server Report:\\\\r\\\\n(?:|\[ *\d+\].*) (?P<bitrate>[0-9\.]+ [KMG]bits\/sec) +(?P<jitter>[0-9\.]+ ms) +(\d+\/..\d+) (\((?P<packetloss>[0-9\.]+)%\))', str(self.ssh.before))
1969 1970 1971 1972 1973 1974
		if result is not None:
			bitrate = result.group('bitrate')
			packetloss = result.group('packetloss')
			jitter = result.group('jitter')
			lock.acquire()
			logging.debug('\u001B[1;37;44m iperf result (' + UE_IPAddress + ') \u001B[0m')
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
1975
			iperfStatus = True
1976 1977
			msg = 'Req Bitrate : ' + req_bandwidth + '\n'
			logging.debug('\u001B[1;34m    Req Bitrate : ' + req_bandwidth + '\u001B[0m')
1978
			if bitrate is not None:
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
1979
				msg += 'Bitrate     : ' + bitrate + '\n'
1980
				logging.debug('\u001B[1;34m    Bitrate     : ' + bitrate + '\u001B[0m')
1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996
				result = re.search('(?P<real_bw>[0-9\.]+) [KMG]bits/sec', str(bitrate))
				if result is not None:
					actual_bw = float(str(result.group('real_bw')))
					result = re.search('[0-9\.]+ K', bitrate)
					if result is not None:
						actual_bw = actual_bw * 1000
					result = re.search('[0-9\.]+ M', bitrate)
					if result is not None:
						actual_bw = actual_bw * 1000000
					result = re.search('[0-9\.]+ G', bitrate)
					if result is not None:
						actual_bw = actual_bw * 1000000000
					br_loss = 100 * actual_bw / req_bw
					bitperf = '%.2f ' % br_loss
					msg += 'Bitrate Perf: ' + bitperf + '%\n'
					logging.debug('\u001B[1;34m    Bitrate Perf: ' + bitperf + '%\u001B[0m')
1997
			if packetloss is not None:
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
1998
				msg += 'Packet Loss : ' + packetloss + '%\n'
1999 2000
				logging.debug('\u001B[1;34m    Packet Loss : ' + packetloss + '%\u001B[0m')
				if float(packetloss) > float(self.iperf_packetloss_threshold):
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
2001
					msg += 'Packet Loss too high!\n'
2002
					logging.debug('\u001B[1;37;41m Packet Loss too high \u001B[0m')
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
2003
					iperfStatus = False
2004
			if jitter is not None:
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
2005
				msg += 'Jitter      : ' + jitter + '\n'
2006
				logging.debug('\u001B[1;34m    Jitter      : ' + jitter + '\u001B[0m')
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
2007 2008 2009 2010 2011 2012 2013
			if (iperfStatus):
				statusQueue.put(0)
			else:
				statusQueue.put(-1)
			statusQueue.put(device_id)
			statusQueue.put(UE_IPAddress)
			statusQueue.put(msg)
2014
			lock.release()
Raphael Defosseux's avatar
Raphael Defosseux committed
2015 2016 2017
			return 0

	def Iperf_analyzeV2Server(self, lock, UE_IPAddress, device_id, statusQueue, iperf_real_options):
2018
		if (not os.path.isfile('iperf_server_' + self.testCase_id + '_' + device_id + '.log')):
Raphael Defosseux's avatar
Raphael Defosseux committed
2019
			self.ping_iperf_wrong_exit(lock, UE_IPAddress, device_id, statusQueue, 'Could not analyze from server log')
Raphael Defosseux's avatar
Raphael Defosseux committed
2020 2021 2022 2023 2024
			return
		# Computing the requested bandwidth in float
		result = re.search('-b (?P<iperf_bandwidth>[0-9\.]+)[KMG]', str(iperf_real_options))
		if result is None:
			logging.debug('Iperf bandwidth Not Found!')
Raphael Defosseux's avatar
Raphael Defosseux committed
2025
			self.ping_iperf_wrong_exit(lock, UE_IPAddress, device_id, statusQueue, 'Could not compute Iperf bandwidth!')
Raphael Defosseux's avatar
Raphael Defosseux committed
2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042
			return
		else:
			req_bandwidth = result.group('iperf_bandwidth')
			req_bw = float(req_bandwidth)
			result = re.search('-b [0-9\.]+K', str(iperf_real_options))
			if result is not None:
				req_bandwidth = '%.1f Kbits/sec' % req_bw
				req_bw = req_bw * 1000
			result = re.search('-b [0-9\.]+M', str(iperf_real_options))
			if result is not None:
				req_bandwidth = '%.1f Mbits/sec' % req_bw
				req_bw = req_bw * 1000000
			result = re.search('-b [0-9\.]+G', str(iperf_real_options))
			if result is not None:
				req_bandwidth = '%.1f Gbits/sec' % req_bw
				req_bw = req_bw * 1000000000

2043
		server_file = open('iperf_server_' + self.testCase_id + '_' + device_id + '.log', 'r')
Raphael Defosseux's avatar
Raphael Defosseux committed
2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110
		br_sum = 0.0
		ji_sum = 0.0
		pl_sum = 0
		ps_sum = 0
		row_idx = 0
		for line in server_file.readlines():
			result = re.search('(?P<bitrate>[0-9\.]+ [KMG]bits\/sec) +(?P<jitter>[0-9\.]+ ms) +(?P<lostPack>[0-9]+)/ +(?P<sentPack>[0-9]+)', str(line))
			if result is not None:
				bitrate = result.group('bitrate')
				jitter = result.group('jitter')
				packetlost = result.group('lostPack')
				packetsent = result.group('sentPack')
				br = bitrate.split(' ')
				ji = jitter.split(' ')
				row_idx = row_idx + 1
				curr_br = float(br[0])
				pl_sum = pl_sum + int(packetlost)
				ps_sum = ps_sum + int(packetsent)
				if (br[1] == 'Kbits/sec'):
					curr_br = curr_br * 1000
				if (br[1] == 'Mbits/sec'):
					curr_br = curr_br * 1000 * 1000
				br_sum = curr_br + br_sum
				ji_sum = float(ji[0]) + ji_sum
		if (row_idx > 0):
			br_sum = br_sum / row_idx
			ji_sum = ji_sum / row_idx
			br_loss = 100 * br_sum / req_bw
			if (br_sum > 1000):
				br_sum = br_sum / 1000
				if (br_sum > 1000):
					br_sum = br_sum / 1000
					bitrate = '%.2f Mbits/sec' % br_sum
				else:
					bitrate = '%.2f Kbits/sec' % br_sum
			else:
				bitrate = '%.2f bits/sec' % br_sum
			bitperf = '%.2f ' % br_loss
			bitperf += '%'
			jitter = '%.2f ms' % (ji_sum)
			if (ps_sum > 0):
				pl = float(100 * pl_sum / ps_sum)
				packetloss = '%2.1f ' % (pl)
				packetloss += '%'
			else:
				packetloss = 'unknown'
			lock.acquire()
			if (br_loss < 90):
				statusQueue.put(1)
			else:
				statusQueue.put(0)
			statusQueue.put(device_id)
			statusQueue.put(UE_IPAddress)
			req_msg = 'Req Bitrate : ' + req_bandwidth
			bir_msg = 'Bitrate     : ' + bitrate
			brl_msg = 'Bitrate Perf: ' + bitperf
			jit_msg = 'Jitter      : ' + jitter
			pal_msg = 'Packet Loss : ' + packetloss
			statusQueue.put(req_msg + '\n' + bir_msg + '\n' + brl_msg + '\n' + jit_msg + '\n' + pal_msg + '\n')
			logging.debug('\u001B[1;37;45m iperf result (' + UE_IPAddress + ') \u001B[0m')
			logging.debug('\u001B[1;35m    ' + req_msg + '\u001B[0m')
			logging.debug('\u001B[1;35m    ' + bir_msg + '\u001B[0m')
			logging.debug('\u001B[1;35m    ' + brl_msg + '\u001B[0m')
			logging.debug('\u001B[1;35m    ' + jit_msg + '\u001B[0m')
			logging.debug('\u001B[1;35m    ' + pal_msg + '\u001B[0m')
			lock.release()
		else:
Raphael Defosseux's avatar
Raphael Defosseux committed
2111
			self.ping_iperf_wrong_exit(lock, UE_IPAddress, device_id, statusQueue, 'Could not analyze from server log')
Raphael Defosseux's avatar
Raphael Defosseux committed
2112 2113 2114

		server_file.close()

2115

Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
2116
	def Iperf_analyzeV3Output(self, lock, UE_IPAddress, device_id, statusQueue):
2117 2118 2119
		result = re.search('(?P<bitrate>[0-9\.]+ [KMG]bits\/sec) +(?:|[0-9\.]+ ms +\d+\/\d+ \((?P<packetloss>[0-9\.]+)%\)) +(?:|receiver)\\\\r\\\\n(?:|\[ *\d+\] Sent \d+ datagrams)\\\\r\\\\niperf Done\.', str(self.ssh.before))
		if result is None:
			result = re.search('(?P<error>iperf: error - [a-zA-Z0-9 :]+)', str(self.ssh.before))
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
2120 2121 2122 2123
			lock.acquire()
			statusQueue.put(-1)
			statusQueue.put(device_id)
			statusQueue.put(UE_IPAddress)
2124 2125
			if result is not None:
				logging.debug('\u001B[1;37;41m ' + result.group('error') + ' \u001B[0m')
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
2126
				statusQueue.put(result.group('error'))
2127 2128
			else:
				logging.debug('\u001B[1;37;41m Bitrate and/or Packet Loss Not Found! \u001B[0m')
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
2129 2130 2131
				statusQueue.put('Bitrate and/or Packet Loss Not Found!')
			lock.release()

2132 2133 2134 2135 2136
		bitrate = result.group('bitrate')
		packetloss = result.group('packetloss')
		lock.acquire()
		logging.debug('\u001B[1;37;44m iperf result (' + UE_IPAddress + ') \u001B[0m')
		logging.debug('\u001B[1;34m    Bitrate     : ' + bitrate + '\u001B[0m')
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
2137 2138
		msg = 'Bitrate     : ' + bitrate + '\n'
		iperfStatus = True
2139 2140
		if packetloss is not None:
			logging.debug('\u001B[1;34m    Packet Loss : ' + packetloss + '%\u001B[0m')
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
2141
			msg += 'Packet Loss : ' + packetloss + '%\n'
2142 2143
			if float(packetloss) > float(self.iperf_packetloss_threshold):
				logging.debug('\u001B[1;37;41m Packet Loss too high \u001B[0m')
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
2144 2145 2146 2147 2148 2149 2150 2151
				msg += 'Packet Loss too high!\n'
				iperfStatus = False
		if (iperfStatus):
			statusQueue.put(0)
		else:
			statusQueue.put(-1)
		statusQueue.put(device_id)
		statusQueue.put(UE_IPAddress)
Raphael Defosseux's avatar
Raphael Defosseux committed
2152
		statusQueue.put(msg)
2153 2154
		lock.release()

Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
2155
	def Iperf_UL_common(self, lock, UE_IPAddress, device_id, idx, ue_num, statusQueue):
2156 2157 2158 2159
		udpIperf = True
		result = re.search('-u', str(self.iperf_args))
		if result is None:
			udpIperf = False
2160 2161 2162 2163 2164 2165 2166 2167
		ipnumbers = UE_IPAddress.split('.')
		if (len(ipnumbers) == 4):
			ipnumbers[3] = '1'
		EPC_Iperf_UE_IPAddress = ipnumbers[0] + '.' + ipnumbers[1] + '.' + ipnumbers[2] + '.' + ipnumbers[3]

		# Launch iperf server on EPC side
		self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
		self.command('cd ' + self.EPCSourceCodePath + '/scripts', '\$', 5)
2168
		self.command('rm -f iperf_server_' + self.testCase_id + '_' + device_id + '.log', '\$', 5)
2169
		port = 5001 + idx
2170
		if udpIperf:
2171
			self.command('echo $USER; nohup iperf -u -s -i 1 -p ' + str(port) + ' > iperf_server_' + self.testCase_id + '_' + device_id + '.log &', self.EPCUserName, 5)
2172
		else:
2173
			self.command('echo $USER; nohup iperf -s -i 1 -p ' + str(port) + ' > iperf_server_' + self.testCase_id + '_' + device_id + '.log &', self.EPCUserName, 5)
2174 2175 2176 2177
		time.sleep(0.5)
		self.close()

		# Launch iperf client on UE
2178 2179 2180 2181 2182 2183
		if (device_id == 'OAI-UE'):
			self.open(self.UEIPAddress, self.UEUserName, self.UEPassword)
			self.command('cd ' + self.UESourceCodePath + '/cmake_targets', '\$', 5)
		else:
			self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
			self.command('cd ' + self.EPCSourceCodePath + '/scripts', '\$', 5)
2184 2185 2186
		iperf_time = self.Iperf_ComputeTime()
		time.sleep(0.5)

2187 2188 2189 2190
		if udpIperf:
			modified_options = self.Iperf_ComputeModifiedBW(idx, ue_num)
		else:
			modified_options = str(self.iperf_args)
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
2191
		modified_options = modified_options.replace('-R','')
2192 2193
		time.sleep(0.5)

2194
		self.command('rm -f iperf_' + self.testCase_id + '_' + device_id + '.log', '\$', 5)
2195 2196 2197 2198
		if (device_id == 'OAI-UE'):
			iperf_status = self.command('iperf -c ' + EPC_Iperf_UE_IPAddress + ' ' + modified_options + ' -p ' + str(port) + ' -B ' + UE_IPAddress + ' 2>&1 | stdbuf -o0 tee iperf_' + self.testCase_id + '_' + device_id + '.log', '\$', int(iperf_time)*5.0)
		else:
			iperf_status = self.command('stdbuf -o0 adb -s ' + device_id + ' shell "/data/local/tmp/iperf -c ' + EPC_Iperf_UE_IPAddress + ' ' + modified_options + ' -p ' + str(port) + '" 2>&1 | stdbuf -o0 tee iperf_' + self.testCase_id + '_' + device_id + '.log', '\$', int(iperf_time)*5.0)
Raphael Defosseux's avatar
Raphael Defosseux committed
2199 2200 2201 2202 2203
		# TIMEOUT Case
		if iperf_status < 0:
			self.close()
			message = 'iperf on UE (' + str(UE_IPAddress) + ') crashed due to TIMEOUT !'
			logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m')
2204
			self.close()
Raphael Defosseux's avatar
Raphael Defosseux committed
2205 2206
			self.ping_iperf_wrong_exit(lock, UE_IPAddress, device_id, statusQueue, message)
			return
2207
		clientStatus = self.Iperf_analyzeV2Output(lock, UE_IPAddress, device_id, statusQueue, modified_options)
Raphael Defosseux's avatar
Raphael Defosseux committed
2208
		self.close()
2209

Raphael Defosseux's avatar
Raphael Defosseux committed
2210
		# Kill iperf server on EPC side
2211 2212 2213
		self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
		self.command('killall --signal SIGKILL iperf', self.EPCUserName, 5)
		self.close()
Raphael Defosseux's avatar
Raphael Defosseux committed
2214
		# in case of failure, retrieve server log
Raphael Defosseux's avatar
Raphael Defosseux committed
2215 2216
		if (clientStatus == -1):
			time.sleep(1)
2217 2218 2219
			if (os.path.isfile('iperf_server_' + self.testCase_id + '_' + device_id + '.log')):
				os.remove('iperf_server_' + self.testCase_id + '_' + device_id + '.log')
			self.copyin(self.EPCIPAddress, self.EPCUserName, self.EPCPassword, self.EPCSourceCodePath + '/scripts/iperf_server_' + self.testCase_id + '_' + device_id + '.log', '.')
Raphael Defosseux's avatar
Raphael Defosseux committed
2220
			self.Iperf_analyzeV2Server(lock, UE_IPAddress, device_id, statusQueue, modified_options)
2221 2222 2223 2224
		# in case of OAI-UE 
		if (device_id == 'OAI-UE'):
			self.copyin(self.UEIPAddress, self.UEUserName, self.UEPassword, self.UESourceCodePath + '/cmake_targets/iperf_' + self.testCase_id + '_' + device_id + '.log', '.')
			self.copyout(self.EPCIPAddress, self.EPCUserName, self.EPCPassword, 'iperf_' + self.testCase_id + '_' + device_id + '.log', self.EPCSourceCodePath + '/scripts')
2225

Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
2226
	def Iperf_common(self, lock, UE_IPAddress, device_id, idx, ue_num, statusQueue):
2227
		try:
2228
			# Single-UE profile -- iperf only on one UE
2229
			if self.iperf_profile == 'single-ue' and idx != 0:
2230
				return
2231
			useIperf3 = False
2232
			udpIperf = True
2233 2234 2235 2236 2237 2238 2239 2240
			if (device_id != 'OAI-UE'):
				self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
				# if by chance ADB server and EPC are on the same remote host, at least log collection will take care of it
				self.command('if [ ! -d ' + self.EPCSourceCodePath + '/scripts ]; then mkdir -p ' + self.EPCSourceCodePath + '/scripts ; fi', '\$', 5)
				self.command('cd ' + self.EPCSourceCodePath + '/scripts', '\$', 5)
				# Checking if iperf / iperf3 are installed
				self.command('adb -s ' + device_id + ' shell "ls /data/local/tmp"', '\$', 5)
				result = re.search('iperf3', str(self.ssh.before))
2241
				if result is None:
2242 2243 2244 2245 2246 2247 2248 2249 2250 2251
					result = re.search('iperf', str(self.ssh.before))
					if result is None:
						message = 'Neither iperf nor iperf3 installed on UE!'
						logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m')
						self.close()
						self.ping_iperf_wrong_exit(lock, UE_IPAddress, device_id, statusQueue, message)
						return
				else:
					useIperf3 = True
				self.close()
2252 2253 2254 2255
			# in case of iperf, UL has its own function
			if (not useIperf3):
				result = re.search('-R', str(self.iperf_args))
				if result is not None:
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
2256
					self.Iperf_UL_common(lock, UE_IPAddress, device_id, idx, ue_num, statusQueue)
2257 2258
					return

2259 2260 2261 2262
			# Launch the IPERF server on the UE side for DL
			if (device_id == 'OAI-UE'):
				self.open(self.UEIPAddress, self.UEUserName, self.UEPassword)
				self.command('cd ' + self.UESourceCodePath + '/cmake_targets', '\$', 5)
2263
				self.command('rm -f iperf_server_' + self.testCase_id + '_' + device_id + '.log', '\$', 5)
2264 2265
				result = re.search('-u', str(self.iperf_args))
				if result is None:
2266
					self.command('echo $USER; nohup iperf -B ' + UE_IPAddress + ' -s -i 1 > iperf_server_' + self.testCase_id + '_' + device_id + '.log &', self.UEUserName, 5)
2267 2268
					udpIperf = False
				else:
2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282
					self.command('echo $USER; nohup iperf -B ' + UE_IPAddress + ' -u -s -i 1 > iperf_server_' + self.testCase_id + '_' + device_id + '.log &', self.UEUserName, 5)
			else:
				self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
				self.command('cd ' + self.EPCSourceCodePath + '/scripts', '\$', 5)
				if (useIperf3):
					self.command('stdbuf -o0 adb -s ' + device_id + ' shell /data/local/tmp/iperf3 -s &', '\$', 5)
				else:
					self.command('rm -f iperf_server_' + self.testCase_id + '_' + device_id + '.log', '\$', 5)
					result = re.search('-u', str(self.iperf_args))
					if result is None:
						self.command('echo $USER; nohup adb -s ' + device_id + ' shell "/data/local/tmp/iperf -s -i 1" > iperf_server_' + self.testCase_id + '_' + device_id + '.log &', self.ADBUserName, 5)
						udpIperf = False
					else:
						self.command('echo $USER; nohup adb -s ' + device_id + ' shell "/data/local/tmp/iperf -u -s -i 1" > iperf_server_' + self.testCase_id + '_' + device_id + '.log &', self.ADBUserName, 5)
2283 2284 2285
			time.sleep(0.5)
			self.close()

2286
			# Launch the IPERF client on the EPC side for DL
2287
			self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
2288 2289
			self.command('cd ' + self.EPCSourceCodePath + '/scripts', '\$', 5)
			iperf_time = self.Iperf_ComputeTime()
2290 2291
			time.sleep(0.5)

2292 2293 2294 2295
			if udpIperf:
				modified_options = self.Iperf_ComputeModifiedBW(idx, ue_num)
			else:
				modified_options = str(self.iperf_args)
2296 2297
			time.sleep(0.5)

2298
			self.command('rm -f iperf_' + self.testCase_id + '_' + device_id + '.log', '\$', 5)
2299
			if (useIperf3):
2300
				self.command('stdbuf -o0 iperf3 -c ' + UE_IPAddress + ' ' + modified_options + ' 2>&1 | stdbuf -o0 tee iperf_' + self.testCase_id + '_' + device_id + '.log', '\$', int(iperf_time)*5.0)
2301

Raphael Defosseux's avatar
Raphael Defosseux committed
2302
				clientStatus = 0
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
2303
				self.Iperf_analyzeV3Output(lock, UE_IPAddress, device_id, statusQueue)
2304
			else:
2305
				iperf_status = self.command('stdbuf -o0 iperf -c ' + UE_IPAddress + ' ' + modified_options + ' 2>&1 | stdbuf -o0 tee iperf_' + self.testCase_id + '_' + device_id + '.log', '\$', int(iperf_time)*5.0)
Raphael Defosseux's avatar
Raphael Defosseux committed
2306 2307 2308 2309 2310 2311
				if iperf_status < 0:
					self.close()
					message = 'iperf on UE (' + str(UE_IPAddress) + ') crashed due to TIMEOUT !'
					logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m')
					self.ping_iperf_wrong_exit(lock, UE_IPAddress, device_id, statusQueue, message)
					return
2312
				clientStatus = self.Iperf_analyzeV2Output(lock, UE_IPAddress, device_id, statusQueue, modified_options)
2313 2314
			self.close()

2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325
			# Kill the IPERF server that runs in background
			if (device_id == 'OAI-UE'):
				self.open(self.UEIPAddress, self.UEUserName, self.UEPassword)
				self.command('killall iperf', '\$', 5)
			else:
				self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
				self.command('stdbuf -o0 adb -s ' + device_id + ' shell ps | grep --color=never iperf | grep -v grep', '\$', 5)
				result = re.search('shell +(?P<pid>\d+)', str(self.ssh.before))
				if result is not None:
					pid_iperf = result.group('pid')
					self.command('stdbuf -o0 adb -s ' + device_id + ' shell kill -KILL ' + pid_iperf, '\$', 5)
2326
			self.close()
2327
			# if the client report is absent, try to analyze the server log file
Raphael Defosseux's avatar
Raphael Defosseux committed
2328 2329
			if (clientStatus == -1):
				time.sleep(1)
2330 2331
				if (os.path.isfile('iperf_server_' + self.testCase_id + '_' + device_id + '.log')):
					os.remove('iperf_server_' + self.testCase_id + '_' + device_id + '.log')
2332 2333 2334 2335
				if (device_id == 'OAI-UE'):
					self.copyin(self.UEIPAddress, self.UEUserName, self.UEPassword, self.UESourceCodePath + '/cmake_targets/iperf_server_' + self.testCase_id + '_' + device_id + '.log', '.')
				else:
					self.copyin(self.ADBIPAddress, self.ADBUserName, self.ADBPassword, self.EPCSourceCodePath + '/scripts/iperf_server_' + self.testCase_id + '_' + device_id + '.log', '.')
Raphael Defosseux's avatar
Raphael Defosseux committed
2336
				self.Iperf_analyzeV2Server(lock, UE_IPAddress, device_id, statusQueue, modified_options)
2337 2338 2339 2340 2341 2342 2343 2344

			# in case of OAI UE: 
			if (device_id == 'OAI-UE'):
				if (os.path.isfile('iperf_server_' + self.testCase_id + '_' + device_id + '.log')):
					pass
				else:
					self.copyin(self.UEIPAddress, self.UEUserName, self.UEPassword, self.UESourceCodePath + '/cmake_targets/iperf_server_' + self.testCase_id + '_' + device_id + '.log', '.')
					self.copyout(self.EPCIPAddress, self.EPCUserName, self.EPCPassword, 'iperf_server_' + self.testCase_id + '_' + device_id + '.log', self.EPCSourceCodePath + '/scripts')
2345 2346 2347
		except:
			os.kill(os.getppid(),signal.SIGUSR1)

2348
	def IperfNoS1(self):
2349 2350 2351 2352 2353 2354 2355 2356
		check_eNB = True
		check_OAI_UE = True
		pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE)
		if (pStatus < 0):
			self.CreateHtmlTestRow(self.iperf_args, 'KO', pStatus)
			self.AutoTerminateUEandeNB()
			self.CreateHtmlTabFooter(False)
			sys.exit(1)
2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374 2375 2376
		if self.eNBIPAddress == '' or self.eNBUserName == '' or self.eNBPassword == '' or self.UEIPAddress == '' or self.UEUserName == '' or self.UEPassword == '':
			Usage()
			sys.exit('Insufficient Parameter')
		server_on_enb = re.search('-R', str(self.iperf_args))
		if server_on_enb is not None:
			iServerIPAddr = self.eNBIPAddress
			iServerUser = self.eNBUserName
			iServerPasswd = self.eNBPassword
			iClientIPAddr = self.UEIPAddress
			iClientUser = self.UEUserName
			iClientPasswd = self.UEPassword
		else:
			iServerIPAddr = self.UEIPAddress
			iServerUser = self.UEUserName
			iServerPasswd = self.UEPassword
			iClientIPAddr = self.eNBIPAddress
			iClientUser = self.eNBUserName
			iClientPasswd = self.eNBPassword
		# Starting the iperf server
		self.open(iServerIPAddr, iServerUser, iServerPasswd)
2377 2378 2379 2380 2381
		# args SHALL be "-c client -u any"
		# -c 10.0.1.2 -u -b 1M -t 30 -i 1 -fm -B 10.0.1.1
		# -B 10.0.1.1 -u -s -i 1 -fm
		server_options = re.sub('-u.*$', '-u -s -i 1 -fm', str(self.iperf_args))
		server_options = server_options.replace('-c','-B')
2382
		self.command('rm -f /tmp/tmp_iperf_server_' + self.testCase_id + '.log', '\$', 5)
2383
		self.command('echo $USER; nohup iperf ' + server_options + ' > /tmp/tmp_iperf_server_' + self.testCase_id + '.log 2>&1 &', iServerUser, 5)
2384 2385 2386 2387 2388 2389 2390 2391
		time.sleep(0.5)
		self.close()

		# Starting the iperf client
		modified_options = self.Iperf_ComputeModifiedBW(0, 1)
		modified_options = modified_options.replace('-R','')
		iperf_time = self.Iperf_ComputeTime()
		self.open(iClientIPAddr, iClientUser, iClientPasswd)
2392 2393
		self.command('rm -f /tmp/tmp_iperf_' + self.testCase_id + '.log', '\$', 5)
		iperf_status = self.command('stdbuf -o0 iperf ' + modified_options + ' 2>&1 | stdbuf -o0 tee /tmp/tmp_iperf_' + self.testCase_id + '.log', '\$', int(iperf_time)*5.0)
2394 2395
		status_queue = SimpleQueue()
		lock = Lock()
2396 2397 2398
		if iperf_status < 0:
			message = 'iperf on OAI UE crashed due to TIMEOUT !'
			logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m')
2399 2400 2401
			clientStatus = -2
		else:
			clientStatus = self.Iperf_analyzeV2Output(lock, '10.0.1.2', 'OAI-UE', status_queue, modified_options)
2402 2403 2404 2405 2406 2407 2408
		self.close()

		# Stopping the iperf server
		self.open(iServerIPAddr, iServerUser, iServerPasswd)
		self.command('killall --signal SIGKILL iperf', '\$', 5)
		time.sleep(0.5)
		self.close()
2409 2410 2411 2412 2413 2414
		if (clientStatus == -1):
			if (os.path.isfile('iperf_server_' + self.testCase_id + '.log')):
				os.remove('iperf_server_' + self.testCase_id + '.log')
			self.copyin(iServerIPAddr, iServerUser, iServerPasswd, '/tmp/tmp_iperf_server_' + self.testCase_id + '.log', 'iperf_server_' + self.testCase_id + '_OAI-UE.log')
			self.Iperf_analyzeV2Server(lock, '10.0.1.2', 'OAI-UE', status_queue, modified_options)

2415 2416 2417 2418 2419 2420 2421
		# copying on the EPC server for logCollection
		copyin_res = self.copyin(iServerIPAddr, iServerUser, iServerPasswd, '/tmp/tmp_iperf_server_' + self.testCase_id + '.log', 'iperf_server_' + self.testCase_id + '_OAI-UE.log')
		if (copyin_res == 0):
			self.copyout(self.EPCIPAddress, self.EPCUserName, self.EPCPassword, 'iperf_server_' + self.testCase_id + '_OAI-UE.log', self.EPCSourceCodePath + '/scripts')
		copyin_res = self.copyin(iClientIPAddr, iClientUser, iClientPasswd, '/tmp/tmp_iperf_' + self.testCase_id + '.log', 'iperf_' + self.testCase_id + '_OAI-UE.log')
		if (copyin_res == 0):
			self.copyout(self.EPCIPAddress, self.EPCUserName, self.EPCPassword, 'iperf_' + self.testCase_id + '_OAI-UE.log', self.EPCSourceCodePath + '/scripts')
2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447
		iperf_noperf = False
		if status_queue.empty():
			iperf_status = False
		else:
			iperf_status = True
		html_queue = SimpleQueue()
		while (not status_queue.empty()):
			count = status_queue.get()
			if (count < 0):
				iperf_status = False
			if (count > 0):
				iperf_noperf = True
			device_id = status_queue.get()
			ip_addr = status_queue.get()
			message = status_queue.get()
			html_cell = '<pre style="background-color:white">UE (' + device_id + ')\nIP Address  : ' + ip_addr + '\n' + message + '</pre>'
			html_queue.put(html_cell)
		if (iperf_noperf and iperf_status):
			self.CreateHtmlTestRowQueue(self.iperf_args, 'PERF NOT MET', len(self.UEDevices), html_queue)
		elif (iperf_status):
			self.CreateHtmlTestRowQueue(self.iperf_args, 'OK', len(self.UEDevices), html_queue)
		else:
			self.CreateHtmlTestRowQueue(self.iperf_args, 'KO', len(self.UEDevices), html_queue)
			self.AutoTerminateUEandeNB()
			self.CreateHtmlTabFooter(False)
			sys.exit(1)
2448

2449
	def Iperf(self):
2450 2451 2452 2453
		result = re.search('noS1', str(self.Initialize_eNB_args))
		if result is not None:
			self.IperfNoS1()
			return
2454 2455 2456
		if self.EPCIPAddress == '' or self.EPCUserName == '' or self.EPCPassword == '' or self.EPCSourceCodePath == '' or self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '':
			Usage()
			sys.exit('Insufficient Parameter')
2457
		check_eNB = True
2458 2459 2460 2461
		if (len(self.UEDevices) == 1) and (self.UEDevices[0] == 'OAI-UE'):
			check_OAI_UE = True
		else:
			check_OAI_UE = False
2462
		pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE)
Raphael Defosseux's avatar
Raphael Defosseux committed
2463 2464
		if (pStatus < 0):
			self.CreateHtmlTestRow(self.iperf_args, 'KO', pStatus)
2465
			self.AutoTerminateUEandeNB()
2466
			self.CreateHtmlTabFooter(False)
Raphael Defosseux's avatar
Raphael Defosseux committed
2467
			sys.exit(1)
2468 2469
		ueIpStatus = self.GetAllUEIPAddresses()
		if (ueIpStatus < 0):
Raphael Defosseux's avatar
Raphael Defosseux committed
2470
			self.CreateHtmlTestRow(self.iperf_args, 'KO', UE_IP_ADDRESS_ISSUE)
2471
			self.AutoTerminateUEandeNB()
2472
			self.CreateHtmlTabFooter(False)
2473
			sys.exit(1)
2474 2475 2476 2477
		multi_jobs = []
		i = 0
		ue_num = len(self.UEIPAddresses)
		lock = Lock()
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
2478
		status_queue = SimpleQueue()
2479 2480
		for UE_IPAddress in self.UEIPAddresses:
			device_id = self.UEDevices[i]
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
2481
			p = Process(target = SSH.Iperf_common, args = (lock,UE_IPAddress,device_id,i,ue_num,status_queue,))
2482 2483 2484 2485 2486 2487
			p.daemon = True
			p.start()
			multi_jobs.append(p)
			i = i + 1
		for job in multi_jobs:
			job.join()
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
2488 2489

		if (status_queue.empty()):
2490
			self.CreateHtmlTestRow(self.iperf_args, 'KO', ALL_PROCESSES_OK)
2491
			self.AutoTerminateUEandeNB()
2492
			self.CreateHtmlTabFooter(False)
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
2493 2494 2495
			sys.exit(1)
		else:
			iperf_status = True
Raphael Defosseux's avatar
Raphael Defosseux committed
2496
			iperf_noperf = False
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
2497 2498 2499 2500 2501
			html_queue = SimpleQueue()
			while (not status_queue.empty()):
				count = status_queue.get()
				if (count < 0):
					iperf_status = False
Raphael Defosseux's avatar
Raphael Defosseux committed
2502 2503
				if (count > 0):
					iperf_noperf = True
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
2504 2505 2506
				device_id = status_queue.get()
				ip_addr = status_queue.get()
				message = status_queue.get()
2507
				html_cell = '<pre style="background-color:white">UE (' + device_id + ')\nIP Address  : ' + ip_addr + '\n' + message + '</pre>'
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
2508
				html_queue.put(html_cell)
Raphael Defosseux's avatar
Raphael Defosseux committed
2509 2510 2511
			if (iperf_noperf and iperf_status):
				self.CreateHtmlTestRowQueue(self.iperf_args, 'PERF NOT MET', len(self.UEDevices), html_queue)
			elif (iperf_status):
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
2512 2513 2514
				self.CreateHtmlTestRowQueue(self.iperf_args, 'OK', len(self.UEDevices), html_queue)
			else:
				self.CreateHtmlTestRowQueue(self.iperf_args, 'KO', len(self.UEDevices), html_queue)
2515
				self.AutoTerminateUEandeNB()
2516
				self.CreateHtmlTabFooter(False)
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
2517
				sys.exit(1)
2518

2519
	def CheckProcessExist(self, check_eNB, check_OAI_UE):
2520
		multi_jobs = []
Raphael Defosseux's avatar
Raphael Defosseux committed
2521
		status_queue = SimpleQueue()
2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540
		# in noS1 config, no need to check status from EPC
		result = re.search('noS1', str(self.Initialize_eNB_args))
		if result is None:
			p = Process(target = SSH.CheckHSSProcess, args = (status_queue,))
			p.daemon = True
			p.start()
			multi_jobs.append(p)
			p = Process(target = SSH.CheckMMEProcess, args = (status_queue,))
			p.daemon = True
			p.start()
			multi_jobs.append(p)
			p = Process(target = SSH.CheckSPGWProcess, args = (status_queue,))
			p.daemon = True
			p.start()
			multi_jobs.append(p)
		else:
			if (check_eNB == False) and (check_OAI_UE == False):
				return 0
		if check_eNB:
Raphael Defosseux's avatar
Raphael Defosseux committed
2541
			p = Process(target = SSH.CheckeNBProcess, args = (status_queue,))
2542 2543 2544
			p.daemon = True
			p.start()
			multi_jobs.append(p)
2545 2546 2547 2548 2549
		if check_OAI_UE:
			p = Process(target = SSH.CheckOAIUEProcess, args = (status_queue,))
			p.daemon = True
			p.start()
			multi_jobs.append(p)
2550 2551 2552
		for job in multi_jobs:
			job.join()

Raphael Defosseux's avatar
Raphael Defosseux committed
2553 2554 2555 2556 2557 2558 2559 2560
		if (status_queue.empty()):
			return -15
		else:
			result = 0
			while (not status_queue.empty()):
				status = status_queue.get()
				if (status < 0):
					result = status
2561
			if result == ENB_PROCESS_FAILED:
2562
				fileCheck = re.search('enb_', str(self.eNBLogFiles[0]))
2563
				if fileCheck is not None:
2564 2565
					self.copyin(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, self.eNBSourceCodePath + '/cmake_targets/' + self.eNBLogFiles[0], '.')
					logStatus = self.AnalyzeLogFile_eNB(self.eNBLogFiles[0])
2566 2567
					if logStatus < 0:
						result = logStatus
2568
					self.eNBLogFiles[0] = ''
2569 2570
				if self.flexranCtrlInstalled and self.flexranCtrlStarted:
					self.TerminateFlexranCtrl()
Raphael Defosseux's avatar
Raphael Defosseux committed
2571 2572
			return result

Boris Djalal's avatar
Boris Djalal committed
2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592
	def CheckOAIUEProcessExist(self, initialize_OAI_UE_flag):
		multi_jobs = []
		status_queue = SimpleQueue()
		if initialize_OAI_UE_flag == False:
			p = Process(target = SSH.CheckOAIUEProcess, args = (status_queue,))
			p.daemon = True
			p.start()
			multi_jobs.append(p)
		for job in multi_jobs:
			job.join()

		if (status_queue.empty()):
			return -15
		else:
			result = 0
			while (not status_queue.empty()):
				status = status_queue.get()
				if (status < 0):
					result = status
			if result == OAI_UE_PROCESS_FAILED:
2593
				fileCheck = re.search('ue_', str(self.UELogFile))
Boris Djalal's avatar
Boris Djalal committed
2594 2595 2596 2597 2598 2599 2600 2601 2602
				if fileCheck is not None:
					self.copyin(self.UEIPAddress, self.UEUserName, self.UEPassword, self.UESourceCodePath + '/cmake_targets/' + self.UELogFile, '.')
					logStatus = self.AnalyzeLogFile_UE(self.UELogFile)
					if logStatus < 0:
						result = logStatus
			return result

	def CheckOAIUEProcess(self, status_queue):
		try:
2603 2604
			self.open(self.UEIPAddress, self.UEUserName, self.UEPassword)
			self.command('stdbuf -o0 ps -aux | grep --color=never softmodem | grep -v grep', '\$', 5)
Boris Djalal's avatar
Boris Djalal committed
2605 2606 2607 2608 2609 2610 2611 2612 2613 2614
			result = re.search('lte-uesoftmodem', str(self.ssh.before))
			if result is None:
				logging.debug('\u001B[1;37;41m OAI UE Process Not Found! \u001B[0m')
				status_queue.put(OAI_UE_PROCESS_FAILED)
			else:
				status_queue.put(OAI_UE_PROCESS_OK)
			self.close()
		except:
			os.kill(os.getppid(),signal.SIGUSR1)

Raphael Defosseux's avatar
Raphael Defosseux committed
2615
	def CheckeNBProcess(self, status_queue):
2616 2617
		try:
			self.open(self.eNBIPAddress, self.eNBUserName, self.eNBPassword)
2618
			self.command('stdbuf -o0 ps -aux | grep --color=never softmodem | grep -v grep', '\$', 5)
2619 2620 2621
			result = re.search('lte-softmodem', str(self.ssh.before))
			if result is None:
				logging.debug('\u001B[1;37;41m eNB Process Not Found! \u001B[0m')
2622
				status_queue.put(ENB_PROCESS_FAILED)
Raphael Defosseux's avatar
Raphael Defosseux committed
2623
			else:
2624
				status_queue.put(ENB_PROCESS_OK)
2625 2626 2627 2628
			self.close()
		except:
			os.kill(os.getppid(),signal.SIGUSR1)

Raphael Defosseux's avatar
Raphael Defosseux committed
2629
	def CheckHSSProcess(self, status_queue):
2630 2631
		try:
			self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
Raphael Defosseux's avatar
Raphael Defosseux committed
2632
			self.command('stdbuf -o0 ps -aux | grep --color=never hss | grep -v grep', '\$', 5)
2633 2634 2635 2636 2637 2638
			if re.match('OAI', self.EPCType, re.IGNORECASE):
				result = re.search('\/bin\/bash .\/run_', str(self.ssh.before))
			else:
				result = re.search('hss_sim s6as diam_hss', str(self.ssh.before))
			if result is None:
				logging.debug('\u001B[1;37;41m HSS Process Not Found! \u001B[0m')
2639
				status_queue.put(HSS_PROCESS_FAILED)
Raphael Defosseux's avatar
Raphael Defosseux committed
2640
			else:
2641
				status_queue.put(HSS_PROCESS_OK)
2642 2643 2644 2645
			self.close()
		except:
			os.kill(os.getppid(),signal.SIGUSR1)

Raphael Defosseux's avatar
Raphael Defosseux committed
2646
	def CheckMMEProcess(self, status_queue):
2647 2648
		try:
			self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
Raphael Defosseux's avatar
Raphael Defosseux committed
2649
			self.command('stdbuf -o0 ps -aux | grep --color=never mme | grep -v grep', '\$', 5)
2650 2651 2652 2653 2654 2655
			if re.match('OAI', self.EPCType, re.IGNORECASE):
				result = re.search('\/bin\/bash .\/run_', str(self.ssh.before))
			else:
				result = re.search('mme', str(self.ssh.before))
			if result is None:
				logging.debug('\u001B[1;37;41m MME Process Not Found! \u001B[0m')
2656
				status_queue.put(MME_PROCESS_FAILED)
Raphael Defosseux's avatar
Raphael Defosseux committed
2657
			else:
2658
				status_queue.put(MME_PROCESS_OK)
2659 2660 2661
			self.close()
		except:
			os.kill(os.getppid(),signal.SIGUSR1)
2662

Raphael Defosseux's avatar
Raphael Defosseux committed
2663
	def CheckSPGWProcess(self, status_queue):
2664 2665 2666
		try:
			self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
			if re.match('OAI', self.EPCType, re.IGNORECASE):
Raphael Defosseux's avatar
Raphael Defosseux committed
2667
				self.command('stdbuf -o0 ps -aux | grep --color=never spgw | grep -v grep', '\$', 5)
2668 2669
				result = re.search('\/bin\/bash .\/run_', str(self.ssh.before))
			else:
Raphael Defosseux's avatar
Raphael Defosseux committed
2670
				self.command('stdbuf -o0 ps -aux | grep --color=never xGw | grep -v grep', '\$', 5)
2671 2672 2673
				result = re.search('xGw', str(self.ssh.before))
			if result is None:
				logging.debug('\u001B[1;37;41m SPGW Process Not Found! \u001B[0m')
2674
				status_queue.put(SPGW_PROCESS_FAILED)
Raphael Defosseux's avatar
Raphael Defosseux committed
2675
			else:
2676
				status_queue.put(SPGW_PROCESS_OK)
2677 2678 2679
			self.close()
		except:
			os.kill(os.getppid(),signal.SIGUSR1)
2680

2681 2682
	def AnalyzeLogFile_eNB(self, eNBlogFile):
		if (not os.path.isfile('./' + eNBlogFile)):
2683
			return -1
2684
		enb_log_file = open('./' + eNBlogFile, 'r')
2685
		exitSignalReceived = False
2686 2687 2688 2689 2690
		foundAssertion = False
		msgAssertion = ''
		msgLine = 0
		foundSegFault = False
		foundRealTimeIssue = False
Raphael Defosseux's avatar
Raphael Defosseux committed
2691 2692 2693 2694 2695 2696 2697
		rrcSetupComplete = 0
		rrcReleaseRequest = 0
		rrcReconfigRequest = 0
		rrcReconfigComplete = 0
		rrcReestablishRequest = 0
		rrcReestablishComplete = 0
		rrcReestablishReject = 0
2698 2699
		rlcDiscardBuffer = 0
		rachCanceledProcedure = 0
Raphael Defosseux's avatar
Raphael Defosseux committed
2700
		uciStatMsgCount = 0
2701 2702
		pdcpFailure = 0
		ulschFailure = 0
2703
		cdrxActivationMessageCount = 0
Raphael Defosseux's avatar
Raphael Defosseux committed
2704
		dropNotEnoughRBs = 0
2705
		self.htmleNBFailureMsg = ''
2706 2707 2708
		isRRU = False
		isSlave = False
		slaveReceivesFrameResyncCmd = False
2709
		for line in enb_log_file.readlines():
2710 2711
			if self.eNBOptions[int(self.eNB_instance)] != '':
				res1 = re.search('max_rxgain (?P<requested_option>[0-9]+)', self.eNBOptions[int(self.eNB_instance)])
2712 2713 2714 2715 2716
				res2 = re.search('max_rxgain (?P<applied_option>[0-9]+)',  str(line))
				if res1 is not None and res2 is not None:
					requested_option = int(res1.group('requested_option'))
					applied_option = int(res2.group('applied_option'))
					if requested_option == applied_option:
2717
						self.htmleNBFailureMsg += '<span class="glyphicon glyphicon-ok-circle"></span> Command line option(s) correctly applied <span class="glyphicon glyphicon-arrow-right"></span> ' + self.eNBOptions[int(self.eNB_instance)] + '\n\n'
2718
					else:
2719
						self.htmleNBFailureMsg += '<span class="glyphicon glyphicon-ban-circle"></span> Command line option(s) NOT applied <span class="glyphicon glyphicon-arrow-right"></span> ' + self.eNBOptions[int(self.eNB_instance)] + '\n\n'
2720
			result = re.search('Exiting OAI softmodem', str(line))
2721
			if result is not None:
2722 2723 2724
				exitSignalReceived = True
			result = re.search('[Ss]egmentation [Ff]ault', str(line))
			if result is not None and not exitSignalReceived:
2725 2726
				foundSegFault = True
			result = re.search('[Cc]ore [dD]ump', str(line))
2727 2728 2729
			if result is not None and not exitSignalReceived:
				foundSegFault = True
			result = re.search('./lte_build_oai/build/lte-softmodem', str(line))
2730
			if result is not None and not exitSignalReceived:
2731 2732
				foundSegFault = True
			result = re.search('[Aa]ssertion', str(line))
2733
			if result is not None and not exitSignalReceived:
2734 2735
				foundAssertion = True
			result = re.search('LLL', str(line))
2736
			if result is not None and not exitSignalReceived:
2737 2738 2739 2740
				foundRealTimeIssue = True
			if foundAssertion and (msgLine < 3):
				msgLine += 1
				msgAssertion += str(line)
2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751
			result = re.search('Setting function for RU', str(line))
			if result is not None:
				isRRU = True
			if isRRU:
				result = re.search('RU 0 is_slave=yes', str(line))
				if result is not None:
					isSlave = True
				if isSlave:
					result = re.search('Received RRU_frame_resynch command', str(line))
					if result is not None:
						slaveReceivesFrameResyncCmd = True
2752
			result = re.search('LTE_RRCConnectionSetupComplete from UE', str(line))
Raphael Defosseux's avatar
Raphael Defosseux committed
2753 2754
			if result is not None:
				rrcSetupComplete += 1
2755
			result = re.search('Generate LTE_RRCConnectionRelease|Generate RRCConnectionRelease', str(line))
Raphael Defosseux's avatar
Raphael Defosseux committed
2756 2757
			if result is not None:
				rrcReleaseRequest += 1
2758
			result = re.search('Generate LTE_RRCConnectionReconfiguration', str(line))
Raphael Defosseux's avatar
Raphael Defosseux committed
2759 2760
			if result is not None:
				rrcReconfigRequest += 1
2761
			result = re.search('LTE_RRCConnectionReconfigurationComplete from UE rnti', str(line))
Raphael Defosseux's avatar
Raphael Defosseux committed
2762 2763
			if result is not None:
				rrcReconfigComplete += 1
2764
			result = re.search('LTE_RRCConnectionReestablishmentRequest', str(line))
Raphael Defosseux's avatar
Raphael Defosseux committed
2765 2766
			if result is not None:
				rrcReestablishRequest += 1
2767
			result = re.search('LTE_RRCConnectionReestablishmentComplete', str(line))
Raphael Defosseux's avatar
Raphael Defosseux committed
2768 2769
			if result is not None:
				rrcReestablishComplete += 1
2770
			result = re.search('LTE_RRCConnectionReestablishmentReject', str(line))
Raphael Defosseux's avatar
Raphael Defosseux committed
2771 2772
			if result is not None:
				rrcReestablishReject += 1
2773 2774 2775
			result = re.search('CDRX configuration activated after RRC Connection', str(line))
			if result is not None:
				cdrxActivationMessageCount += 1
Raphael Defosseux's avatar
Raphael Defosseux committed
2776 2777 2778
			result = re.search('uci->stat', str(line))
			if result is not None:
				uciStatMsgCount += 1
2779 2780 2781 2782 2783 2784
			result = re.search('PDCP.*Out of Resources.*reason', str(line))
			if result is not None:
				pdcpFailure += 1
			result = re.search('ULSCH in error in round', str(line))
			if result is not None:
				ulschFailure += 1
2785 2786 2787 2788 2789 2790
			result = re.search('BAD all_segments_received', str(line))
			if result is not None:
				rlcDiscardBuffer += 1
			result = re.search('Canceled RA procedure for UE rnti', str(line))
			if result is not None:
				rachCanceledProcedure += 1
Raphael Defosseux's avatar
Raphael Defosseux committed
2791 2792 2793
			result = re.search('dropping, not enough RBs', str(line))
			if result is not None:
				dropNotEnoughRBs += 1
2794
		enb_log_file.close()
2795
		logging.debug('   File analysis completed')
Raphael Defosseux's avatar
Raphael Defosseux committed
2796
		if uciStatMsgCount > 0:
2797 2798 2799 2800 2801
			statMsg = 'eNB showed ' + str(uciStatMsgCount) + ' "uci->stat" message(s)'
			logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
			self.htmleNBFailureMsg += statMsg + '\n'
		if pdcpFailure > 0:
			statMsg = 'eNB showed ' + str(pdcpFailure) + ' "PDCP Out of Resources" message(s)'
Raphael Defosseux's avatar
Raphael Defosseux committed
2802 2803
			logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
			self.htmleNBFailureMsg += statMsg + '\n'
2804 2805 2806
		if ulschFailure > 0:
			statMsg = 'eNB showed ' + str(ulschFailure) + ' "ULSCH in error in round" message(s)'
			logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
2807
			self.htmleNBFailureMsg += statMsg + '\n'
Raphael Defosseux's avatar
Raphael Defosseux committed
2808 2809 2810 2811
		if dropNotEnoughRBs > 0:
			statMsg = 'eNB showed ' + str(dropNotEnoughRBs) + ' "dropping, not enough RBs" message(s)'
			logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
			self.htmleNBFailureMsg += statMsg + '\n'
2812 2813
		if rrcSetupComplete > 0:
			rrcMsg = 'eNB completed ' + str(rrcSetupComplete) + ' RRC Connection Setup(s)'
Raphael Defosseux's avatar
Raphael Defosseux committed
2814 2815 2816 2817 2818 2819 2820 2821
			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
			self.htmleNBFailureMsg += rrcMsg + '\n'
		if rrcReleaseRequest > 0:
			rrcMsg = 'eNB requested ' + str(rrcReleaseRequest) + ' RRC Connection Release(s)'
			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
			self.htmleNBFailureMsg += rrcMsg + '\n'
		if rrcReconfigRequest > 0 or rrcReconfigComplete > 0:
			rrcMsg = 'eNB requested ' + str(rrcReconfigRequest) + ' RRC Connection Reconfiguration(s)'
2822
			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
2823 2824 2825 2826 2827 2828 2829 2830 2831 2832 2833 2834 2835 2836
			self.htmleNBFailureMsg += rrcMsg + '\n'
			rrcMsg = ' -- ' + str(rrcReconfigComplete) + ' were completed'
			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
			self.htmleNBFailureMsg += rrcMsg + '\n'
		if rrcReestablishRequest > 0 or rrcReestablishComplete > 0 or rrcReestablishReject > 0:
			rrcMsg = 'eNB requested ' + str(rrcReestablishRequest) + ' RRC Connection Reestablishment(s)'
			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
			self.htmleNBFailureMsg += rrcMsg + '\n'
			rrcMsg = ' -- ' + str(rrcReestablishComplete) + ' were completed'
			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
			self.htmleNBFailureMsg += rrcMsg + '\n'
			rrcMsg = ' -- ' + str(rrcReestablishReject) + ' were rejected'
			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
			self.htmleNBFailureMsg += rrcMsg + '\n'
2837 2838 2839 2840
		if cdrxActivationMessageCount > 0:
			rrcMsg = 'eNB activated the CDRX Configuration for ' + str(cdrxActivationMessageCount) + ' time(s)'
			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
			self.htmleNBFailureMsg += rrcMsg + '\n'
2841 2842 2843 2844
		if rachCanceledProcedure > 0:
			rachMsg = 'eNB cancelled ' + str(rachCanceledProcedure) + ' RA procedure(s)'
			logging.debug('\u001B[1;30;43m ' + rachMsg + ' \u001B[0m')
			self.htmleNBFailureMsg += rachMsg + '\n'
2845 2846 2847 2848 2849 2850 2851 2852 2853 2854 2855
		if isRRU:
			if isSlave:
				if slaveReceivesFrameResyncCmd:
					rruMsg = 'Slave RRU received the RRU_frame_resynch command from RAU'
					logging.debug('\u001B[1;30;43m ' + rruMsg + ' \u001B[0m')
					self.htmleNBFailureMsg += rruMsg + '\n'
				else:
					rruMsg = 'Slave RRU DID NOT receive the RRU_frame_resynch command from RAU'
					logging.debug('\u001B[1;37;41m ' + rruMsg + ' \u001B[0m')
					self.htmleNBFailureMsg += rruMsg + '\n'
					return ENB_PROCESS_SLAVE_RRU_NOT_SYNCED
2856 2857 2858 2859 2860
		if foundSegFault:
			logging.debug('\u001B[1;37;41m eNB ended with a Segmentation Fault! \u001B[0m')
			return ENB_PROCESS_SEG_FAULT
		if foundAssertion:
			logging.debug('\u001B[1;37;41m eNB ended with an assertion! \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
2861
			self.htmleNBFailureMsg += msgAssertion
2862 2863 2864
			return ENB_PROCESS_ASSERTION
		if foundRealTimeIssue:
			logging.debug('\u001B[1;37;41m eNB faced real time issues! \u001B[0m')
2865 2866
			self.htmleNBFailureMsg += 'eNB faced real time issues!\n'
			#return ENB_PROCESS_REALTIME_ISSUE
2867 2868 2869 2870 2871
		if rlcDiscardBuffer > 0:
			rlcMsg = 'eNB RLC discarded ' + str(rlcDiscardBuffer) + ' buffer(s)'
			logging.debug('\u001B[1;37;41m ' + rlcMsg + ' \u001B[0m')
			self.htmleNBFailureMsg += rlcMsg + '\n'
			return ENB_PROCESS_REALTIME_ISSUE
2872 2873
		return 0

Boris Djalal's avatar
Boris Djalal committed
2874 2875 2876 2877
	def AnalyzeLogFile_UE(self, UElogFile):
		if (not os.path.isfile('./' + UElogFile)):
			return -1
		ue_log_file = open('./' + UElogFile, 'r')
2878
		exitSignalReceived = False
Boris Djalal's avatar
Boris Djalal committed
2879 2880 2881 2882 2883 2884
		foundAssertion = False
		msgAssertion = ''
		msgLine = 0
		foundSegFault = False
		foundRealTimeIssue = False
		uciStatMsgCount = 0
2885 2886 2887
		pdcpDataReqFailedCount = 0
		badDciCount = 0
		rrcConnectionRecfgComplete = 0
2888
		no_cell_sync_found = False
2889 2890
		mib_found = False
		frequency_found = False
2891
		plmn_found = False
2892
		self.htmlUEFailureMsg = ''
Boris Djalal's avatar
Boris Djalal committed
2893
		for line in ue_log_file.readlines():
2894
			result = re.search('Exiting OAI softmodem', str(line))
Boris Djalal's avatar
Boris Djalal committed
2895
			if result is not None:
2896
				exitSignalReceived = True
2897
			result = re.search('System error|[Ss]egmentation [Ff]ault|======= Backtrace: =========|======= Memory map: ========', str(line))
2898
			if result is not None and not exitSignalReceived:
Boris Djalal's avatar
Boris Djalal committed
2899 2900
				foundSegFault = True
			result = re.search('[Cc]ore [dD]ump', str(line))
2901 2902 2903
			if result is not None and not exitSignalReceived:
				foundSegFault = True
			result = re.search('./lte-uesoftmodem', str(line))
2904
			if result is not None and not exitSignalReceived:
Boris Djalal's avatar
Boris Djalal committed
2905 2906
				foundSegFault = True
			result = re.search('[Aa]ssertion', str(line))
2907
			if result is not None and not exitSignalReceived:
Boris Djalal's avatar
Boris Djalal committed
2908 2909
				foundAssertion = True
			result = re.search('LLL', str(line))
2910
			if result is not None and not exitSignalReceived:
Boris Djalal's avatar
Boris Djalal committed
2911 2912 2913 2914 2915
				foundRealTimeIssue = True
			if foundAssertion and (msgLine < 3):
				msgLine += 1
				msgAssertion += str(line)
			result = re.search('uci->stat', str(line))
2916
			if result is not None and not exitSignalReceived:
Boris Djalal's avatar
Boris Djalal committed
2917
				uciStatMsgCount += 1
2918 2919 2920 2921 2922 2923 2924 2925 2926
			result = re.search('PDCP data request failed', str(line))
			if result is not None and not exitSignalReceived:
				pdcpDataReqFailedCount += 1
			result = re.search('bad DCI 1A', str(line))
			if result is not None and not exitSignalReceived:
				badDciCount += 1
			result = re.search('Generating RRCConnectionReconfigurationComplete', str(line))
			if result is not None:
				rrcConnectionRecfgComplete += 1
Boris Djalal's avatar
Boris Djalal committed
2927
			# No cell synchronization found, abandoning
2928 2929 2930
			result = re.search('No cell synchronization found, abandoning', str(line))
			if result is not None:
				no_cell_sync_found = True
2931
			result = re.search("MIB Information => ([a-zA-Z]{1,10}), ([a-zA-Z]{1,10}), NidCell (?P<nidcell>\d{1,3}), N_RB_DL (?P<n_rb_dl>\d{1,3}), PHICH DURATION (?P<phich_duration>\d), PHICH RESOURCE (?P<phich_resource>.{1,4}), TX_ANT (?P<tx_ant>\d)", str(line))
2932
			if result is not None and (not mib_found):
2933
				try:
2934 2935 2936 2937 2938 2939 2940 2941 2942 2943 2944 2945 2946 2947 2948 2949 2950 2951 2952
					mibMsg = "MIB Information: " + result.group(1) + ', ' + result.group(2)
					self.htmlUEFailureMsg += mibMsg + '\n'
					logging.debug('\033[94m' + mibMsg + '\033[0m')
					mibMsg = "    nidcell = " + result.group('nidcell')
					self.htmlUEFailureMsg += mibMsg
					logging.debug('\033[94m' + mibMsg + '\033[0m')
					mibMsg = "    n_rb_dl = " + result.group('n_rb_dl')
					self.htmlUEFailureMsg += mibMsg + '\n'
					logging.debug('\033[94m' + mibMsg + '\033[0m')
					mibMsg = "    phich_duration = " + result.group('phich_duration')
					self.htmlUEFailureMsg += mibMsg
					logging.debug('\033[94m' + mibMsg + '\033[0m')
					mibMsg = "    phich_resource = " + result.group('phich_resource')
					self.htmlUEFailureMsg += mibMsg + '\n'
					logging.debug('\033[94m' + mibMsg + '\033[0m')
					mibMsg = "    tx_ant = " + result.group('tx_ant')
					self.htmlUEFailureMsg += mibMsg + '\n'
					logging.debug('\033[94m' + mibMsg + '\033[0m')
					mib_found = True
2953
				except Exception as e:
2954
					logging.error('\033[91m' + "MIB marker was not found" + '\033[0m')
2955
			result = re.search("Measured Carrier Frequency (?P<measured_carrier_frequency>\d{1,15}) Hz", str(line))
2956
			if result is not None and (not frequency_found):
2957
				try:
2958 2959 2960 2961
					mibMsg = "Measured Carrier Frequency = " + result.group('measured_carrier_frequency') + ' Hz'
					self.htmlUEFailureMsg += mibMsg + '\n'
					logging.debug('\033[94m' + mibMsg + '\033[0m')
					frequency_found = True
2962
				except Exception as e:
2963
					logging.error('\033[91m' + "Measured Carrier Frequency not found" + '\033[0m')
2964 2965 2966 2967 2968 2969 2970 2971 2972
			result = re.search("PLMN MCC (?P<mcc>\d{1,3}), MNC (?P<mnc>\d{1,3}), TAC", str(line))
			if result is not None and (not plmn_found):
				try:
					mibMsg = 'PLMN MCC = ' + result.group('mcc') + ' MNC = ' + result.group('mnc')
					self.htmlUEFailureMsg += mibMsg + '\n'
					logging.debug('\033[94m' + mibMsg + '\033[0m')
					plmn_found = True
				except Exception as e:
					logging.error('\033[91m' + "PLMN not found" + '\033[0m')
2973 2974 2975
			result = re.search("Found (?P<operator>[\w,\s]{1,15}) \(name from internal table\)", str(line))
			if result is not None:
				try:
2976 2977 2978
					mibMsg = "The operator is: " + result.group('operator')
					self.htmlUEFailureMsg += mibMsg + '\n'
					logging.debug('\033[94m' + mibMsg + '\033[0m')
2979
				except Exception as e:
2980
					logging.error('\033[91m' + "Operator name not found" + '\033[0m')
2981 2982 2983
			result = re.search("SIB5 InterFreqCarrierFreq element (.{1,4})/(.{1,4})", str(line))
			if result is not None:
				try:
2984 2985 2986
					mibMsg = "SIB5 InterFreqCarrierFreq element " + result.group(1) + '/' + result.group(2)
					self.htmlUEFailureMsg += mibMsg + ' -> '
					logging.debug('\033[94m' + mibMsg + '\033[0m')
2987
				except Exception as e:
2988 2989
					logging.error('\033[91m' + "SIB5 InterFreqCarrierFreq element not found" + '\033[0m')
			result = re.search("DL Carrier Frequency/ARFCN : (?P<carrier_frequency>\d{1,15}/\d{1,4})", str(line))
2990 2991
			if result is not None:
				try:
2992 2993 2994 2995 2996
					freq = result.group('carrier_frequency')
					new_freq = re.sub('/[0-9]+','',freq)
					float_freq = float(new_freq) / 1000000
					self.htmlUEFailureMsg += 'DL Freq: ' + ('%.1f' % float_freq) + ' MHz'
					logging.debug('\033[94m' + "    DL Carrier Frequency is: " + freq + '\033[0m')
2997
				except Exception as e:
2998
					logging.error('\033[91m' + "    DL Carrier Frequency not found" + '\033[0m')
2999 3000 3001
			result = re.search("AllowedMeasBandwidth : (?P<allowed_bandwidth>\d{1,7})", str(line))
			if result is not None:
				try:
3002 3003 3004
					prb = result.group('allowed_bandwidth')
					self.htmlUEFailureMsg += ' -- PRB: ' + prb + '\n'
					logging.debug('\033[94m' + "    AllowedMeasBandwidth: " + prb + '\033[0m')
3005
				except Exception as e:
3006
					logging.error('\033[91m' + "    AllowedMeasBandwidth not found" + '\033[0m')
Boris Djalal's avatar
Boris Djalal committed
3007
		ue_log_file.close()
3008 3009 3010 3011
		if rrcConnectionRecfgComplete > 0:
			statMsg = 'UE connected to eNB (' + str(rrcConnectionRecfgComplete) + ' RRCConnectionReconfigurationComplete message(s) generated)'
			logging.debug('\033[94m' + statMsg + '\033[0m')
			self.htmlUEFailureMsg += statMsg + '\n'
Boris Djalal's avatar
Boris Djalal committed
3012 3013 3014 3015
		if uciStatMsgCount > 0:
			statMsg = 'UE showed ' + str(uciStatMsgCount) + ' "uci->stat" message(s)'
			logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
			self.htmlUEFailureMsg += statMsg + '\n'
3016 3017
		if pdcpDataReqFailedCount > 0:
			statMsg = 'UE showed ' + str(pdcpDataReqFailedCount) + ' "PDCP data request failed" message(s)'
Boris Djalal's avatar
Boris Djalal committed
3018 3019
			logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
			self.htmlUEFailureMsg += statMsg + '\n'
3020 3021
		if badDciCount > 0:
			statMsg = 'UE showed ' + str(badDciCount) + ' "bad DCI 1A" message(s)'
Boris Djalal's avatar
Boris Djalal committed
3022 3023 3024 3025 3026 3027
			logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
			self.htmlUEFailureMsg += statMsg + '\n'
		if foundSegFault:
			logging.debug('\u001B[1;37;41m UE ended with a Segmentation Fault! \u001B[0m')
			return ENB_PROCESS_SEG_FAULT
		if foundAssertion:
3028 3029
			logging.debug('\u001B[1;30;43m UE showed an assertion! \u001B[0m')
			self.htmlUEFailureMsg += 'UE showed an assertion!\n'
3030
			if not mib_found or not frequency_found:
3031
				return OAI_UE_PROCESS_ASSERTION
Boris Djalal's avatar
Boris Djalal committed
3032 3033 3034 3035
		if foundRealTimeIssue:
			logging.debug('\u001B[1;37;41m UE faced real time issues! \u001B[0m')
			self.htmlUEFailureMsg += 'UE faced real time issues!\n'
			#return ENB_PROCESS_REALTIME_ISSUE
3036 3037 3038
		if no_cell_sync_found and not mib_found:
			logging.debug('\u001B[1;37;41m UE could not synchronize ! \u001B[0m')
			self.htmlUEFailureMsg += 'UE could not synchronize!\n'
3039
			return OAI_UE_PROCESS_COULD_NOT_SYNC
Boris Djalal's avatar
Boris Djalal committed
3040 3041
		return 0

3042
	def TerminateeNB(self):
3043 3044 3045 3046 3047 3048 3049 3050 3051 3052 3053 3054 3055 3056 3057 3058 3059 3060 3061 3062
		if self.eNB_serverId == '0':
			lIpAddr = self.eNBIPAddress
			lUserName = self.eNBUserName
			lPassWord = self.eNBPassword
			lSourcePath = self.eNBSourceCodePath
		elif self.eNB_serverId == '1':
			lIpAddr = self.eNB1IPAddress
			lUserName = self.eNB1UserName
			lPassWord = self.eNB1Password
			lSourcePath = self.eNB1SourceCodePath
		elif self.eNB_serverId == '2':
			lIpAddr = self.eNB2IPAddress
			lUserName = self.eNB2UserName
			lPassWord = self.eNB2Password
			lSourcePath = self.eNB2SourceCodePath
		if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '':
			Usage()
			sys.exit('Insufficient Parameter')
		self.open(lIpAddr, lUserName, lPassWord)
		self.command('cd ' + lSourcePath + '/cmake_targets', '\$', 5)
3063
		self.command('stdbuf -o0  ps -aux | grep --color=never softmodem | grep -v grep', '\$', 5)
3064 3065
		result = re.search('lte-softmodem', str(self.ssh.before))
		if result is not None:
3066 3067
			self.command('echo ' + lPassWord + ' | sudo -S daemon --name=enb' + str(self.eNB_instance) + '_daemon --stop', '\$', 5)
			self.command('echo ' + lPassWord + ' | sudo -S killall --signal SIGINT lte-softmodem || true', '\$', 5)
3068
			time.sleep(5)
3069 3070 3071
			self.command('stdbuf -o0  ps -aux | grep --color=never softmodem | grep -v grep', '\$', 5)
			result = re.search('lte-softmodem', str(self.ssh.before))
			if result is not None:
3072
				self.command('echo ' + lPassWord + ' | sudo -S killall --signal SIGKILL lte-softmodem || true', '\$', 5)
3073 3074
				time.sleep(2)
		self.command('rm -f my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5)
3075
		self.close()
3076 3077 3078 3079 3080 3081 3082
		# If tracer options is on, stopping tshark on EPC side
		result = re.search('T_stdout', str(self.Initialize_eNB_args))
		if result is not None:
			self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
			logging.debug('\u001B[1m Stopping tshark \u001B[0m')
			self.command('echo ' + self.EPCPassword + ' | sudo -S killall --signal SIGKILL tshark', '\$', 5)
			time.sleep(1)
3083 3084 3085
			if self.EPC_PcapFileName != '':
				self.command('echo ' + self.EPCPassword + ' | sudo -S chmod 666 /tmp/' + self.EPC_PcapFileName, '\$', 5)
				self.copyin(self.EPCIPAddress, self.EPCUserName, self.EPCPassword, '/tmp/' + self.EPC_PcapFileName, '.')
3086
				self.copyout(lIpAddr, lUserName, lPassWord, self.EPC_PcapFileName, lSourcePath + '/cmake_targets/.')
3087
			self.close()
3088
			logging.debug('\u001B[1m Replaying RAW record file\u001B[0m')
3089 3090 3091 3092 3093 3094 3095 3096 3097 3098
			self.open(lIpAddr, lUserName, lPassWord)
			self.command('cd ' + lSourcePath + '/common/utils/T/tracer/', '\$', 5)
			enbLogFile = self.eNBLogFiles[int(self.eNB_instance)]
			raw_record_file = enbLogFile.replace('.log', '_record.raw')
			replay_log_file = enbLogFile.replace('.log', '_replay.log')
			extracted_txt_file = enbLogFile.replace('.log', '_extracted_messages.txt')
			extracted_log_file = enbLogFile.replace('.log', '_extracted_messages.log')
			self.command('./extract_config -i ' + lSourcePath + '/cmake_targets/' + raw_record_file + ' > ' + lSourcePath + '/cmake_targets/' + extracted_txt_file, '\$', 5)
			self.command('echo $USER; nohup ./replay -i ' + lSourcePath + '/cmake_targets/' + raw_record_file + ' > ' + lSourcePath + '/cmake_targets/' + replay_log_file + ' 2>&1 &', lUserName, 5)
			self.command('./textlog -d ' +  lSourcePath + '/cmake_targets/' + extracted_txt_file + ' -no-gui -ON -full > ' + lSourcePath + '/cmake_targets/' + extracted_log_file, '\$', 5)
3099
			self.close()
3100
			self.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/' + extracted_log_file, '.')
3101 3102 3103
			logging.debug('\u001B[1m Analyzing eNB replay logfile \u001B[0m')
			logStatus = self.AnalyzeLogFile_eNB(extracted_log_file)
			self.CreateHtmlTestRow('N/A', 'OK', ALL_PROCESSES_OK)
3104
			self.eNBLogFiles[int(self.eNB_instance)] = ''
3105
		else:
3106
			analyzeFile = False
3107
			if self.eNBLogFiles[int(self.eNB_instance)] != '':
3108
				analyzeFile = True
3109 3110
				fileToAnalyze = self.eNBLogFiles[int(self.eNB_instance)]
				self.eNBLogFiles[int(self.eNB_instance)] = ''
3111
			if analyzeFile:
3112
				copyin_res = self.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/' + fileToAnalyze, '.')
3113 3114 3115 3116 3117
				if (copyin_res == -1):
					logging.debug('\u001B[1;37;41m Could not copy eNB logfile to analyze it! \u001B[0m')
					self.htmleNBFailureMsg = 'Could not copy eNB logfile to analyze it!'
					self.CreateHtmlTestRow('N/A', 'KO', ENB_PROCESS_NOLOGFILE_TO_ANALYZE)
					return
3118 3119
				if self.eNB_serverId != '0':
					self.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, './' + fileToAnalyze, self.eNBSourceCodePath + '/cmake_targets/')
Raphael Defosseux's avatar
Raphael Defosseux committed
3120
				logging.debug('\u001B[1m Analyzing eNB logfile \u001B[0m ' + fileToAnalyze)
3121
				logStatus = self.AnalyzeLogFile_eNB(fileToAnalyze)
3122 3123
				if (logStatus < 0):
					self.CreateHtmlTestRow('N/A', 'KO', logStatus)
3124
					self.CreateHtmlTabFooter(False)
3125 3126 3127 3128 3129
					sys.exit(1)
				else:
					self.CreateHtmlTestRow('N/A', 'OK', ALL_PROCESSES_OK)
			else:
				self.CreateHtmlTestRow('N/A', 'OK', ALL_PROCESSES_OK)
3130 3131 3132 3133 3134 3135

	def TerminateHSS(self):
		self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
		if re.match('OAI', self.EPCType, re.IGNORECASE):
			self.command('echo ' + self.EPCPassword + ' | sudo -S killall --signal SIGINT run_hss oai_hss || true', '\$', 5)
			time.sleep(2)
Raphael Defosseux's avatar
Raphael Defosseux committed
3136
			self.command('stdbuf -o0  ps -aux | grep hss | grep -v grep', '\$', 5)
3137 3138 3139 3140
			result = re.search('\/bin\/bash .\/run_', str(self.ssh.before))
			if result is not None:
				self.command('echo ' + self.EPCPassword + ' | sudo -S killall --signal SIGKILL run_hss oai_hss || true', '\$', 5)
		else:
3141 3142
			self.command('cd ' + self.EPCSourceCodePath, '\$', 5)
			self.command('cd scripts', '\$', 5)
3143
			self.command('echo ' + self.EPCPassword + ' | sudo -S daemon --name=simulated_hss --stop', '\$', 5)
3144 3145
			time.sleep(1)
			self.command('echo ' + self.EPCPassword + ' | sudo -S killall --signal SIGKILL hss_sim', '\$', 5)
3146
		self.close()
3147
		self.CreateHtmlTestRow('N/A', 'OK', ALL_PROCESSES_OK)
3148 3149 3150 3151 3152 3153

	def TerminateMME(self):
		self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
		if re.match('OAI', self.EPCType, re.IGNORECASE):
			self.command('echo ' + self.EPCPassword + ' | sudo -S killall --signal SIGINT run_mme mme || true', '\$', 5)
			time.sleep(2)
Raphael Defosseux's avatar
Raphael Defosseux committed
3154
			self.command('stdbuf -o0 ps -aux | grep mme | grep -v grep', '\$', 5)
3155 3156 3157 3158
			result = re.search('\/bin\/bash .\/run_', str(self.ssh.before))
			if result is not None:
				self.command('echo ' + self.EPCPassword + ' | sudo -S killall --signal SIGKILL run_mme mme || true', '\$', 5)
		else:
3159
			self.command('cd /opt/ltebox/tools', '\$', 5)
3160 3161
			self.command('echo ' + self.EPCPassword + ' | sudo -S ./stop_mme', '\$', 5)
		self.close()
3162
		self.CreateHtmlTestRow('N/A', 'OK', ALL_PROCESSES_OK)
3163 3164 3165 3166 3167 3168

	def TerminateSPGW(self):
		self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
		if re.match('OAI', self.EPCType, re.IGNORECASE):
			self.command('echo ' + self.EPCPassword + ' | sudo -S killall --signal SIGINT run_spgw spgw || true', '\$', 5)
			time.sleep(2)
Raphael Defosseux's avatar
Raphael Defosseux committed
3169
			self.command('stdbuf -o0 ps -aux | grep spgw | grep -v grep', '\$', 5)
3170 3171 3172 3173
			result = re.search('\/bin\/bash .\/run_', str(self.ssh.before))
			if result is not None:
				self.command('echo ' + self.EPCPassword + ' | sudo -S killall --signal SIGKILL run_spgw spgw || true', '\$', 5)
		else:
3174
			self.command('cd /opt/ltebox/tools', '\$', 5)
3175 3176
			self.command('echo ' + self.EPCPassword + ' | sudo -S ./stop_xGw', '\$', 5)
		self.close()
3177
		self.CreateHtmlTestRow('N/A', 'OK', ALL_PROCESSES_OK)
3178

3179 3180 3181 3182 3183 3184 3185 3186 3187 3188 3189 3190 3191 3192 3193
	def TerminateFlexranCtrl(self):
		if self.flexranCtrlInstalled == False or self.flexranCtrlStarted == False:
			return
		if self.EPCIPAddress == '' or self.EPCUserName == '' or self.EPCPassword == '':
			Usage()
			sys.exit('Insufficient Parameter')
		self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
		self.command('echo ' + self.EPCPassword + ' | sudo -S daemon --name=flexran_rtc_daemon --stop', '\$', 5)
		time.sleep(1)
		self.command('echo ' + self.EPCPassword + ' | sudo -S killall --signal SIGKILL rt_controller', '\$', 5)
		time.sleep(1)
		self.close()
		self.flexranCtrlStarted = False
		self.CreateHtmlTestRow('N/A', 'OK', ALL_PROCESSES_OK)

3194 3195 3196
	def TerminateUE_common(self, device_id):
		try:
			self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
3197
			# back in airplane mode on (ie radio off)
3198 3199 3200
			self.command('stdbuf -o0 adb -s ' + device_id + ' shell /data/local/tmp/off', '\$', 60)
			logging.debug('\u001B[1mUE (' + device_id + ') Detach Completed\u001B[0m')

3201 3202 3203 3204 3205
			self.command('stdbuf -o0 adb -s ' + device_id + ' shell ps | grep --color=never iperf | grep -v grep', '\$', 5)
			result = re.search('shell +(?P<pid>\d+)', str(self.ssh.before))
			if result is not None:
				pid_iperf = result.group('pid')
				self.command('stdbuf -o0 adb -s ' + device_id + ' shell kill -KILL ' + pid_iperf, '\$', 5)
3206 3207 3208 3209 3210 3211
			self.close()
		except:
			os.kill(os.getppid(),signal.SIGUSR1)

	def TerminateUE(self):
		terminate_ue_flag = True
3212
		self.GetAllUEDevices(terminate_ue_flag)
3213 3214 3215 3216 3217 3218 3219 3220
		multi_jobs = []
		for device_id in self.UEDevices:
			p = Process(target= SSH.TerminateUE_common, args = (device_id,))
			p.daemon = True
			p.start()
			multi_jobs.append(p)
		for job in multi_jobs:
			job.join()
3221
		self.CreateHtmlTestRow('N/A', 'OK', ALL_PROCESSES_OK)
3222

3223 3224 3225
	def TerminateOAIUE(self):
		self.open(self.UEIPAddress, self.UEUserName, self.UEPassword)
		self.command('cd ' + self.UESourceCodePath + '/cmake_targets', '\$', 5)
3226
		self.command('ps -aux | grep --color=never softmodem | grep -v grep', '\$', 5)
3227 3228
		result = re.search('lte-uesoftmodem', str(self.ssh.before))
		if result is not None:
3229 3230
			self.command('echo ' + self.UEPassword + ' | sudo -S daemon --name=ue' + str(self.UE_instance) + '_daemon --stop', '\$', 5)
			self.command('echo ' + self.UEPassword + ' | sudo -S killall --signal SIGINT lte-uesoftmodem || true', '\$', 5)
3231
			time.sleep(5)
3232 3233 3234 3235 3236 3237
			self.command('ps -aux | grep --color=never softmodem | grep -v grep', '\$', 5)
			result = re.search('lte-uesoftmodem', str(self.ssh.before))
			if result is not None:
				self.command('echo ' + self.UEPassword + ' | sudo -S killall --signal SIGKILL lte-uesoftmodem || true', '\$', 5)
				time.sleep(2)
		self.command('rm -f my-lte-uesoftmodem-run' + str(self.UE_instance) + '.sh', '\$', 5)
3238
		self.close()
3239
		result = re.search('ue_', str(self.UELogFile))
3240
		if result is not None:
3241 3242 3243 3244
			copyin_res = self.copyin(self.UEIPAddress, self.UEUserName, self.UEPassword, self.UESourceCodePath + '/cmake_targets/' + self.UELogFile, '.')
			if (copyin_res == -1):
				logging.debug('\u001B[1;37;41m Could not copy UE logfile to analyze it! \u001B[0m')
				self.htmlUEFailureMsg = 'Could not copy UE logfile to analyze it!'
3245
				self.CreateHtmlTestRow('N/A', 'KO', OAI_UE_PROCESS_NOLOGFILE_TO_ANALYZE, 'UE')
3246
				self.UELogFile = ''
3247 3248 3249 3250 3251 3252
				return
			logging.debug('\u001B[1m Analyzing UE logfile \u001B[0m')
			logStatus = self.AnalyzeLogFile_UE(self.UELogFile)
			result = re.search('--no-L2-connect', str(self.Initialize_OAI_UE_args))
			if result is not None:
				ueAction = 'Sniffing'
3253
			else:
3254 3255 3256 3257 3258
				ueAction = 'Connection'
			if (logStatus < 0):
				logging.debug('\u001B[1m' + ueAction + ' Failed \u001B[0m')
				self.htmlUEFailureMsg = '<b>' + ueAction + ' Failed</b>\n' + self.htmlUEFailureMsg
				self.CreateHtmlTestRow('N/A', 'KO', logStatus, 'UE')
3259 3260
				# In case of sniffing on commercial eNBs we have random results
				# Not an error then
3261 3262 3263
				if (logStatus != OAI_UE_PROCESS_COULD_NOT_SYNC) or (ueAction != 'Sniffing'):
					self.Initialize_OAI_UE_args = ''
					self.AutoTerminateUEandeNB()
3264 3265
					self.CreateHtmlTabFooter(False)
					sys.exit(1)
3266 3267 3268 3269 3270 3271 3272
			else:
				logging.debug('\u001B[1m' + ueAction + ' Completed \u001B[0m')
				self.htmlUEFailureMsg = '<b>' + ueAction + ' Completed</b>\n' + self.htmlUEFailureMsg
				self.CreateHtmlTestRow('N/A', 'OK', ALL_PROCESSES_OK)
			self.UELogFile = ''
		else:
			self.CreateHtmlTestRow('N/A', 'OK', ALL_PROCESSES_OK)
3273

3274
	def AutoTerminateUEandeNB(self):
3275 3276 3277 3278 3279 3280 3281 3282 3283 3284 3285 3286 3287 3288 3289 3290
		if (self.ADBIPAddress != 'none'):
			self.testCase_id = 'AUTO-KILL-UE'
			self.desc = 'Automatic Termination of UE'
			self.ShowTestID()
			self.TerminateUE()
		if (self.Initialize_OAI_UE_args != ''):
			self.testCase_id = 'AUTO-KILL-UE'
			self.desc = 'Automatic Termination of UE'
			self.ShowTestID()
			self.TerminateOAIUE()
		if (self.Initialize_eNB_args != ''):
			self.testCase_id = 'AUTO-KILL-eNB'
			self.desc = 'Automatic Termination of eNB'
			self.ShowTestID()
			self.eNB_instance = '0'
			self.TerminateeNB()
3291

3292 3293 3294 3295
	def IdleSleep(self):
		time.sleep(self.idle_sleep_time)
		self.CreateHtmlTestRow(str(self.idle_sleep_time) + ' sec', 'OK', ALL_PROCESSES_OK)

3296
	def LogCollectBuild(self):
Boris Djalal's avatar
Boris Djalal committed
3297
		if (self.eNBIPAddress != '' and self.eNBUserName != '' and self.eNBPassword != ''):
3298 3299 3300 3301
			IPAddress = self.eNBIPAddress
			UserName = self.eNBUserName
			Password = self.eNBPassword
			SourceCodePath = self.eNBSourceCodePath
Boris Djalal's avatar
Boris Djalal committed
3302
		elif (self.UEIPAddress != '' and self.UEUserName != '' and self.UEPassword != ''):
3303 3304 3305 3306
			IPAddress = self.UEIPAddress
			UserName = self.UEUserName
			Password = self.UEPassword
			SourceCodePath = self.UESourceCodePath
Boris Djalal's avatar
fix  
Boris Djalal committed
3307 3308
		else:
			sys.exit('Insufficient Parameter')
3309 3310
		self.open(IPAddress, UserName, Password)
		self.command('cd ' + SourceCodePath, '\$', 5)
3311
		self.command('cd cmake_targets', '\$', 5)
Raphael Defosseux's avatar
Raphael Defosseux committed
3312
		self.command('rm -f build.log.zip', '\$', 5)
3313 3314
		self.command('zip build.log.zip build_log_*/*', '\$', 60)
		self.close()
3315

3316 3317 3318 3319
	def LogCollecteNB(self):
		self.open(self.eNBIPAddress, self.eNBUserName, self.eNBPassword)
		self.command('cd ' + self.eNBSourceCodePath, '\$', 5)
		self.command('cd cmake_targets', '\$', 5)
3320
		self.command('echo ' + self.eNBPassword + ' | sudo -S rm -f enb.log.zip', '\$', 5)
3321 3322
		self.command('echo ' + self.eNBPassword + ' | sudo -S zip enb.log.zip enb*.log core* enb_*record.raw enb_*.pcap enb_*txt', '\$', 60)
		self.command('echo ' + self.eNBPassword + ' | sudo -S rm enb*.log core* enb_*record.raw enb_*.pcap enb_*txt', '\$', 5)
3323
		self.close()
3324

3325 3326 3327 3328
	def LogCollectPing(self):
		self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
		self.command('cd ' + self.EPCSourceCodePath, '\$', 5)
		self.command('cd scripts', '\$', 5)
Raphael Defosseux's avatar
Raphael Defosseux committed
3329
		self.command('rm -f ping.log.zip', '\$', 5)
3330 3331 3332
		self.command('zip ping.log.zip ping*.log', '\$', 60)
		self.command('rm ping*.log', '\$', 5)
		self.close()
3333

3334 3335 3336 3337
	def LogCollectIperf(self):
		self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
		self.command('cd ' + self.EPCSourceCodePath, '\$', 5)
		self.command('cd scripts', '\$', 5)
Raphael Defosseux's avatar
Raphael Defosseux committed
3338
		self.command('rm -f iperf.log.zip', '\$', 5)
3339 3340 3341
		self.command('zip iperf.log.zip iperf*.log', '\$', 60)
		self.command('rm iperf*.log', '\$', 5)
		self.close()
3342

3343 3344
	def LogCollectHSS(self):
		self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
3345 3346
		self.command('cd ' + self.EPCSourceCodePath, '\$', 5)
		self.command('cd scripts', '\$', 5)
Raphael Defosseux's avatar
Raphael Defosseux committed
3347
		self.command('rm -f hss.log.zip', '\$', 5)
3348 3349 3350
		if re.match('OAI', self.EPCType, re.IGNORECASE):
			self.command('zip hss.log.zip hss*.log', '\$', 60)
			self.command('rm hss*.log', '\$', 5)
3351 3352 3353
		else:
			self.command('cp /opt/hss_sim0609/hss.log .', '\$', 60)
			self.command('zip hss.log.zip hss.log', '\$', 60)
3354
		self.close()
3355

3356 3357 3358
	def LogCollectMME(self):
		self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
		self.command('cd ' + self.EPCSourceCodePath, '\$', 5)
3359
		self.command('cd scripts', '\$', 5)
Raphael Defosseux's avatar
Raphael Defosseux committed
3360
		self.command('rm -f mme.log.zip', '\$', 5)
3361 3362 3363 3364
		if re.match('OAI', self.EPCType, re.IGNORECASE):
			self.command('zip mme.log.zip mme*.log', '\$', 60)
			self.command('rm mme*.log', '\$', 5)
		else:
3365 3366
			self.command('cp /opt/ltebox/var/log/*Log.0 .', '\$', 5)
			self.command('zip mme.log.zip mmeLog.0 s1apcLog.0 s1apsLog.0 s11cLog.0 libLog.0 s1apCodecLog.0', '\$', 60)
3367
		self.close()
3368

3369 3370 3371
	def LogCollectSPGW(self):
		self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
		self.command('cd ' + self.EPCSourceCodePath, '\$', 5)
3372
		self.command('cd scripts', '\$', 5)
Raphael Defosseux's avatar
Raphael Defosseux committed
3373
		self.command('rm -f spgw.log.zip', '\$', 5)
3374 3375 3376 3377
		if re.match('OAI', self.EPCType, re.IGNORECASE):
			self.command('zip spgw.log.zip spgw*.log', '\$', 60)
			self.command('rm spgw*.log', '\$', 5)
		else:
3378 3379
			self.command('cp /opt/ltebox/var/log/xGwLog.0 .', '\$', 5)
			self.command('zip spgw.log.zip xGwLog.0', '\$', 60)
3380
		self.close()
3381

Boris Djalal's avatar
Boris Djalal committed
3382 3383 3384 3385 3386 3387 3388 3389 3390
	def LogCollectOAIUE(self):
		self.open(self.UEIPAddress, self.UEUserName, self.UEPassword)
		self.command('cd ' + self.UESourceCodePath, '\$', 5)
		self.command('cd cmake_targets', '\$', 5)
		self.command('echo ' + self.UEPassword + ' | sudo -S rm -f ue.log.zip', '\$', 5)
		self.command('echo ' + self.UEPassword + ' | sudo -S zip ue.log.zip ue*.log core* ue_*record.raw ue_*.pcap ue_*txt', '\$', 60)
		self.command('echo ' + self.UEPassword + ' | sudo -S rm ue*.log core* ue_*record.raw ue_*.pcap ue_*txt', '\$', 5)
		self.close()

3391 3392
	def RetrieveSystemVersion(self, machine):
		if self.eNBIPAddress == 'none' or self.UEIPAddress == 'none':
3393 3394 3395
			self.OsVersion = 'Ubuntu 16.04.5 LTS'
			self.KernelVersion = '4.15.0-45-generic'
			self.UhdVersion = '3.13.0.1-0'
3396
			self.UsrpBoard = 'B210'
3397 3398 3399
			self.CpuNb = '4'
			self.CpuModel = 'Intel(R) Core(TM) i5-6200U'
			self.CpuMHz = '2399.996 MHz'
3400 3401 3402 3403 3404 3405 3406 3407 3408 3409 3410 3411 3412 3413 3414 3415
			return 0
		if machine == 'eNB':
			if self.eNBIPAddress != '' and self.eNBUserName != '' and self.eNBPassword != '':
				IPAddress = self.eNBIPAddress
				UserName = self.eNBUserName
				Password = self.eNBPassword
			else:
				return -1
		if machine == 'UE':
			if self.UEIPAddress != '' and self.UEUserName != '' and self.UEPassword != '':
				IPAddress = self.UEIPAddress
				UserName = self.UEUserName
				Password = self.UEPassword
			else:
				return -1

3416
		self.open(IPAddress, UserName, Password)
3417 3418 3419
		self.command('lsb_release -a', '\$', 5)
		result = re.search('Description:\\\\t(?P<os_type>[a-zA-Z0-9\-\_\.\ ]+)', str(self.ssh.before))
		if result is not None:
3420 3421
			self.OsVersion = result.group('os_type')
			logging.debug('OS is: ' + self.OsVersion)
3422 3423 3424
		self.command('uname -r', '\$', 5)
		result = re.search('uname -r\\\\r\\\\n(?P<kernel_version>[a-zA-Z0-9\-\_\.]+)', str(self.ssh.before))
		if result is not None:
3425 3426
			self.KernelVersion = result.group('kernel_version')
			logging.debug('Kernel Version is: ' + self.KernelVersion)
3427 3428
		self.command('dpkg --list | egrep --color=never libuhd003', '\$', 5)
		result = re.search('libuhd003:amd64 *(?P<uhd_version>[0-9\.]+)', str(self.ssh.before))
3429
		if result is not None:
3430 3431
			self.UhdVersion = result.group('uhd_version')
			logging.debug('UHD Version is: ' + self.UhdVersion)
3432
		self.command('echo ' + Password + ' | sudo -S uhd_find_devices', '\$', 15)
3433 3434
		result = re.search('product: (?P<usrp_board>[0-9A-Za-z]+)\\\\r\\\\n', str(self.ssh.before))
		if result is not None:
3435 3436
			self.UsrpBoard = result.group('usrp_board')
			logging.debug('USRP Board  is: ' + self.UsrpBoard)
3437 3438 3439
		self.command('lscpu', '\$', 5)
		result = re.search('CPU\(s\): *(?P<nb_cpus>[0-9]+).*Model name: *(?P<model>[a-zA-Z0-9\-\_\.\ \(\)]+).*CPU MHz: *(?P<cpu_mhz>[0-9\.]+)', str(self.ssh.before))
		if result is not None:
3440 3441 3442 3443 3444 3445
			self.CpuNb = result.group('nb_cpus')
			logging.debug('nb_cpus: ' + self.CpuNb)
			self.CpuModel = result.group('model')
			logging.debug('model: ' + self.CpuModel)
			self.CpuMHz = result.group('cpu_mhz') + ' MHz'
			logging.debug('cpu_mhz: ' + self.CpuMHz)
3446 3447
		self.close()

3448
#-----------------------------------------------------------
Raphael Defosseux's avatar
Raphael Defosseux committed
3449 3450 3451 3452
# HTML Reporting....
#-----------------------------------------------------------
	def CreateHtmlHeader(self):
		if (not self.htmlHeaderCreated):
3453 3454 3455
			logging.debug('\u001B[1m----------------------------------------\u001B[0m')
			logging.debug('\u001B[1m  Creating HTML header \u001B[0m')
			logging.debug('\u001B[1m----------------------------------------\u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
3456 3457 3458 3459
			self.htmlFile = open('test_results.html', 'w')
			self.htmlFile.write('<!DOCTYPE html>\n')
			self.htmlFile.write('<html class="no-js" lang="en-US">\n')
			self.htmlFile.write('<head>\n')
3460 3461 3462 3463
			self.htmlFile.write('  <meta name="viewport" content="width=device-width, initial-scale=1">\n')
			self.htmlFile.write('  <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/css/bootstrap.min.css">\n')
			self.htmlFile.write('  <script src="https://ajax.googleapis.com/ajax/libs/jquery/3.3.1/jquery.min.js"></script>\n')
			self.htmlFile.write('  <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/js/bootstrap.min.js"></script>\n')
Raphael Defosseux's avatar
Raphael Defosseux committed
3464 3465
			self.htmlFile.write('  <title>Test Results for TEMPLATE_JOB_NAME job build #TEMPLATE_BUILD_ID</title>\n')
			self.htmlFile.write('</head>\n')
3466
			self.htmlFile.write('<body><div class="container">\n')
3467
			self.htmlFile.write('  <br>\n')
Raphael Defosseux's avatar
Raphael Defosseux committed
3468 3469 3470 3471
			self.htmlFile.write('  <table style="border-collapse: collapse; border: none;">\n')
			self.htmlFile.write('    <tr style="border-collapse: collapse; border: none;">\n')
			self.htmlFile.write('      <td style="border-collapse: collapse; border: none;">\n')
			self.htmlFile.write('        <a href="http://www.openairinterface.org/">\n')
3472
			self.htmlFile.write('           <img src="http://www.openairinterface.org/wp-content/uploads/2016/03/cropped-oai_final_logo2.png" alt="" border="none" height=50 width=150>\n')
Raphael Defosseux's avatar
Raphael Defosseux committed
3473 3474 3475 3476 3477 3478 3479 3480 3481
			self.htmlFile.write('           </img>\n')
			self.htmlFile.write('        </a>\n')
			self.htmlFile.write('      </td>\n')
			self.htmlFile.write('      <td style="border-collapse: collapse; border: none; vertical-align: center;">\n')
			self.htmlFile.write('        <b><font size = "6">Job Summary -- Job: TEMPLATE_JOB_NAME -- Build-ID: TEMPLATE_BUILD_ID</font></b>\n')
			self.htmlFile.write('      </td>\n')
			self.htmlFile.write('    </tr>\n')
			self.htmlFile.write('  </table>\n')
			self.htmlFile.write('  <br>\n')
3482
			self.htmlFile.write('  <div class="alert alert-info"><strong> <span class="glyphicon glyphicon-dashboard"></span> TEMPLATE_STAGE_NAME</strong></div>\n')
Raphael Defosseux's avatar
Raphael Defosseux committed
3483 3484
			self.htmlFile.write('  <table border = "1">\n')
			self.htmlFile.write('     <tr>\n')
3485
			self.htmlFile.write('       <td bgcolor = "lightcyan" > <span class="glyphicon glyphicon-time"></span> Build Start Time (UTC) </td>\n')
3486 3487 3488
			self.htmlFile.write('       <td>TEMPLATE_BUILD_TIME</td>\n')
			self.htmlFile.write('     </tr>\n')
			self.htmlFile.write('     <tr>\n')
3489
			self.htmlFile.write('       <td bgcolor = "lightcyan" > <span class="glyphicon glyphicon-cloud-upload"></span> GIT Repository </td>\n')
3490
			self.htmlFile.write('       <td><a href="' + self.ranRepository + '">' + self.ranRepository + '</a></td>\n')
Raphael Defosseux's avatar
Raphael Defosseux committed
3491 3492
			self.htmlFile.write('     </tr>\n')
			self.htmlFile.write('     <tr>\n')
3493
			self.htmlFile.write('       <td bgcolor = "lightcyan" > <span class="glyphicon glyphicon-wrench"></span> Job Trigger </td>\n')
3494
			if (self.ranAllowMerge):
Raphael Defosseux's avatar
Raphael Defosseux committed
3495 3496 3497 3498 3499
				self.htmlFile.write('       <td>Merge-Request</td>\n')
			else:
				self.htmlFile.write('       <td>Push to Branch</td>\n')
			self.htmlFile.write('     </tr>\n')
			self.htmlFile.write('     <tr>\n')
3500
			if (self.ranAllowMerge):
3501
				self.htmlFile.write('       <td bgcolor = "lightcyan" > <span class="glyphicon glyphicon-log-out"></span> Source Branch </td>\n')
Raphael Defosseux's avatar
Raphael Defosseux committed
3502
			else:
3503
				self.htmlFile.write('       <td bgcolor = "lightcyan" > <span class="glyphicon glyphicon-tree-deciduous"></span> Branch</td>\n')
3504
			self.htmlFile.write('       <td>' + self.ranBranch + '</td>\n')
Raphael Defosseux's avatar
Raphael Defosseux committed
3505 3506
			self.htmlFile.write('     </tr>\n')
			self.htmlFile.write('     <tr>\n')
3507
			if (self.ranAllowMerge):
3508
				self.htmlFile.write('       <td bgcolor = "lightcyan" > <span class="glyphicon glyphicon-tag"></span> Source Commit ID </td>\n')
Raphael Defosseux's avatar
Raphael Defosseux committed
3509
			else:
3510
				self.htmlFile.write('       <td bgcolor = "lightcyan" > <span class="glyphicon glyphicon-tag"></span> Commit ID </td>\n')
3511
			self.htmlFile.write('       <td>' + self.ranCommitID + '</td>\n')
Raphael Defosseux's avatar
Raphael Defosseux committed
3512
			self.htmlFile.write('     </tr>\n')
3513 3514
			if self.ranAllowMerge != '':
				commit_message = subprocess.check_output("git log -n1 --pretty=format:\"%s\" " + self.ranCommitID, shell=True, universal_newlines=True)
3515 3516
				commit_message = commit_message.strip()
				self.htmlFile.write('     <tr>\n')
3517
				if (self.ranAllowMerge):
3518 3519 3520 3521 3522
					self.htmlFile.write('       <td bgcolor = "lightcyan" > <span class="glyphicon glyphicon-comment"></span> Source Commit Message </td>\n')
				else:
					self.htmlFile.write('       <td bgcolor = "lightcyan" > <span class="glyphicon glyphicon-comment"></span> Commit Message </td>\n')
				self.htmlFile.write('       <td>' + commit_message + '</td>\n')
				self.htmlFile.write('     </tr>\n')
3523
			if (self.ranAllowMerge):
Raphael Defosseux's avatar
Raphael Defosseux committed
3524
				self.htmlFile.write('     <tr>\n')
3525
				self.htmlFile.write('       <td bgcolor = "lightcyan" > <span class="glyphicon glyphicon-log-in"></span> Target Branch </td>\n')
3526
				if (self.ranTargetBranch == ''):
3527 3528
					self.htmlFile.write('       <td>develop</td>\n')
				else:
3529
					self.htmlFile.write('       <td>' + self.ranTargetBranch + '</td>\n')
Raphael Defosseux's avatar
Raphael Defosseux committed
3530 3531 3532
				self.htmlFile.write('     </tr>\n')
			self.htmlFile.write('  </table>\n')

3533
			if (self.ADBIPAddress != 'none'):
Boris Djalal's avatar
Boris Djalal committed
3534
				terminate_ue_flag = True
3535 3536
				self.GetAllUEDevices(terminate_ue_flag)
				self.GetAllCatMDevices(terminate_ue_flag)
3537 3538 3539
				self.htmlUEConnected = len(self.UEDevices)
				self.htmlFile.write('  <h2><span class="glyphicon glyphicon-phone"></span> <span class="glyphicon glyphicon-menu-right"></span> ' + str(len(self.UEDevices)) + ' UE(s) is(are) connected to ADB bench server</h2>\n')
				self.htmlFile.write('  <h2><span class="glyphicon glyphicon-phone"></span> <span class="glyphicon glyphicon-menu-right"></span> ' + str(len(self.CatMDevices)) + ' CAT-M UE(s) is(are) connected to bench server</h2>\n')
3540 3541 3542 3543
			else:
				self.UEDevices.append('OAI-UE')
				self.htmlUEConnected = len(self.UEDevices)
				self.htmlFile.write('  <h2><span class="glyphicon glyphicon-phone"></span> <span class="glyphicon glyphicon-menu-right"></span> ' + str(len(self.UEDevices)) + ' OAI UE(s) is(are) connected to CI bench</h2>\n')
3544
			self.htmlFile.write('  <br>\n')
3545 3546 3547 3548 3549 3550 3551 3552 3553 3554 3555 3556 3557 3558 3559
			self.htmlFile.write('  <ul class="nav nav-pills">\n')
			count = 0
			while (count < self.nbTestXMLfiles):
				pillMsg = '    <li><a data-toggle="pill" href="#'
				pillMsg += self.htmlTabRefs[count]
				pillMsg += '">'
				pillMsg += self.htmlTabNames[count]
				pillMsg += ' <span class="glyphicon glyphicon-'
				pillMsg += self.htmlTabIcons[count]
				pillMsg += '"></span></a></li>\n'
				self.htmlFile.write(pillMsg)
				count += 1
			self.htmlFile.write('  </ul>\n')
			self.htmlFile.write('  <div class="tab-content">\n')
			self.htmlFile.close()
3560

3561 3562 3563 3564 3565 3566 3567 3568 3569 3570 3571
	def CreateHtmlTabHeader(self):
		if (not self.htmlHeaderCreated):
			if (not os.path.isfile('test_results.html')):
				self.CreateHtmlHeader()
			self.htmlFile = open('test_results.html', 'a')
			if (self.nbTestXMLfiles == 1):
				self.htmlFile.write('  <div id="' + self.htmlTabRefs[0] + '" class="tab-pane fade">\n')
				self.htmlFile.write('  <h3>Test Summary for <span class="glyphicon glyphicon-file"></span> ' + self.testXMLfiles[0] + '</h3>\n')
			else:
				self.htmlFile.write('  <div id="build-tab" class="tab-pane fade">\n')
			self.htmlFile.write('  <table class="table" border = "1">\n')
Raphael Defosseux's avatar
Raphael Defosseux committed
3572 3573 3574 3575 3576
			self.htmlFile.write('      <tr bgcolor = "#33CCFF" >\n')
			self.htmlFile.write('        <th>Test Id</th>\n')
			self.htmlFile.write('        <th>Test Desc</th>\n')
			self.htmlFile.write('        <th>Test Options</th>\n')
			self.htmlFile.write('        <th>Test Status</th>\n')
3577 3578 3579 3580 3581
			if (self.htmlUEConnected == -1):
				terminate_ue_flag = True
				if (self.ADBIPAddress != 'none'):
					self.GetAllUEDevices(terminate_ue_flag)
					self.GetAllCatMDevices(terminate_ue_flag)
3582 3583
				else:
					self.UEDevices.append('OAI-UE')
3584 3585
				self.htmlUEConnected = len(self.UEDevices)

Raphael Defosseux's avatar
Raphael Defosseux committed
3586 3587 3588 3589 3590 3591 3592
			i = 0
			while (i < self.htmlUEConnected):
				self.htmlFile.write('        <th>UE' + str(i) + ' Status</th>\n')
				i += 1
			self.htmlFile.write('      </tr>\n')
		self.htmlHeaderCreated = True

3593
	def CreateHtmlTabFooter(self, passStatus):
Raphael Defosseux's avatar
Raphael Defosseux committed
3594
		if ((not self.htmlFooterCreated) and (self.htmlHeaderCreated)):
3595
			self.htmlFile.write('      <tr>\n')
3596
			self.htmlFile.write('        <th bgcolor = "#33CCFF" colspan=2>Final Tab Status</th>\n')
3597
			if passStatus:
3598
				self.htmlFile.write('        <th bgcolor = "green" colspan=' + str(2 + self.htmlUEConnected) + '><font color="white">PASS <span class="glyphicon glyphicon-ok"></span> </font></th>\n')
3599
			else:
3600
				self.htmlFile.write('        <th bgcolor = "red" colspan=' + str(2 + self.htmlUEConnected) + '><font color="white">FAIL <span class="glyphicon glyphicon-remove"></span> </font></th>\n')
3601
			self.htmlFile.write('      </tr>\n')
Raphael Defosseux's avatar
Raphael Defosseux committed
3602
			self.htmlFile.write('  </table>\n')
3603 3604 3605 3606 3607
			self.htmlFile.write('  </div>\n')
		self.htmlFooterCreated = False

	def CreateHtmlFooter(self, passStatus):
		if (os.path.isfile('test_results.html')):
3608 3609 3610
			logging.debug('\u001B[1m----------------------------------------\u001B[0m')
			logging.debug('\u001B[1m  Creating HTML footer \u001B[0m')
			logging.debug('\u001B[1m----------------------------------------\u001B[0m')
3611

3612 3613 3614
			self.htmlFile = open('test_results.html', 'a')
			self.htmlFile.write('</div>\n')
			self.htmlFile.write('  <p></p>\n')
3615
			self.htmlFile.write('  <table class="table table-condensed">\n')
3616 3617 3618 3619 3620 3621 3622 3623 3624 3625 3626 3627 3628 3629 3630 3631 3632 3633 3634 3635 3636 3637 3638 3639 3640 3641 3642 3643 3644

			machines = [ 'eNB', 'UE' ]
			for machine in machines:
				res = self.RetrieveSystemVersion(machine)
				if res == -1:
					continue
				self.htmlFile.write('      <tr>\n')
				self.htmlFile.write('        <th colspan=8>' + str(machine) + ' Server Characteristics</th>\n')
				self.htmlFile.write('      </tr>\n')
				self.htmlFile.write('      <tr>\n')
				self.htmlFile.write('        <td>OS Version</td>\n')
				self.htmlFile.write('        <td><span class="label label-default">' + self.OsVersion + '</span></td>\n')
				self.htmlFile.write('        <td>Kernel Version</td>\n')
				self.htmlFile.write('        <td><span class="label label-default">' + self.KernelVersion + '</span></td>\n')
				self.htmlFile.write('        <td>UHD Version</td>\n')
				self.htmlFile.write('        <td><span class="label label-default">' + self.UhdVersion + '</span></td>\n')
				self.htmlFile.write('        <td>USRP Board</td>\n')
				self.htmlFile.write('        <td><span class="label label-default">' + self.UsrpBoard + '</span></td>\n')
				self.htmlFile.write('      </tr>\n')
				self.htmlFile.write('      <tr>\n')
				self.htmlFile.write('        <td>Nb CPUs</td>\n')
				self.htmlFile.write('        <td><span class="label label-default">' + self.CpuNb + '</span></td>\n')
				self.htmlFile.write('        <td>CPU Model Name</td>\n')
				self.htmlFile.write('        <td><span class="label label-default">' + self.CpuModel + '</span></td>\n')
				self.htmlFile.write('        <td>CPU Frequency</td>\n')
				self.htmlFile.write('        <td><span class="label label-default">' + self.CpuMHz + '</span></td>\n')
				self.htmlFile.write('        <td></td>\n')
				self.htmlFile.write('        <td></td>\n')
				self.htmlFile.write('      </tr>\n')
3645
			self.htmlFile.write('      <tr>\n')
3646
			self.htmlFile.write('        <th colspan=5 bgcolor = "#33CCFF">Final Status</th>\n')
3647
			if passStatus:
3648
				self.htmlFile.write('        <th colspan=3 bgcolor="green"><font color="white">PASS <span class="glyphicon glyphicon-ok"></span></font></th>\n')
3649
			else:
3650
				self.htmlFile.write('        <th colspan=3 bgcolor="red"><font color="white">FAIL <span class="glyphicon glyphicon-remove"></span> </font></th>\n')
3651
			self.htmlFile.write('      </tr>\n')
3652
			self.htmlFile.write('  </table>\n')
3653 3654
			self.htmlFile.write('  <p></p>\n')
			self.htmlFile.write('  <div class="well well-lg">End of Test Report -- Copyright <span class="glyphicon glyphicon-copyright-mark"></span> 2018 <a href="http://www.openairinterface.org/">OpenAirInterface</a>. All Rights Reserved.</div>\n')
3655
			self.htmlFile.write('</div></body>\n')
Raphael Defosseux's avatar
Raphael Defosseux committed
3656 3657 3658
			self.htmlFile.write('</html>\n')
			self.htmlFile.close()

3659
	def CreateHtmlTestRow(self, options, status, processesStatus, machine='eNB'):
Raphael Defosseux's avatar
Raphael Defosseux committed
3660 3661
		if ((not self.htmlFooterCreated) and (self.htmlHeaderCreated)):
			self.htmlFile.write('      <tr>\n')
3662 3663
			self.htmlFile.write('        <td bgcolor = "lightcyan" >' + self.testCase_id  + '</td>\n')
			self.htmlFile.write('        <td>' + self.desc  + '</td>\n')
Raphael Defosseux's avatar
Raphael Defosseux committed
3664 3665 3666 3667
			self.htmlFile.write('        <td>' + str(options)  + '</td>\n')
			if (str(status) == 'OK'):
				self.htmlFile.write('        <td bgcolor = "lightgreen" >' + str(status)  + '</td>\n')
			elif (str(status) == 'KO'):
Raphael Defosseux's avatar
Raphael Defosseux committed
3668 3669
				if (processesStatus == 0):
					self.htmlFile.write('        <td bgcolor = "lightcoral" >' + str(status)  + '</td>\n')
3670
				elif (processesStatus == ENB_PROCESS_FAILED):
3671 3672 3673
					self.htmlFile.write('        <td bgcolor = "lightcoral" >KO - eNB process not found</td>\n')
				elif (processesStatus == OAI_UE_PROCESS_FAILED):
					self.htmlFile.write('        <td bgcolor = "lightcoral" >KO - OAI UE process not found</td>\n')
3674
				elif (processesStatus == ENB_PROCESS_SEG_FAULT):
3675
					self.htmlFile.write('        <td bgcolor = "lightcoral" >KO - ' + machine + ' process ended in Segmentation Fault</td>\n')
3676
				elif (processesStatus == ENB_PROCESS_ASSERTION) or (processesStatus == OAI_UE_PROCESS_ASSERTION):
3677
					self.htmlFile.write('        <td bgcolor = "lightcoral" >KO - ' + machine + ' process ended in Assertion</td>\n')
3678
				elif (processesStatus == ENB_PROCESS_REALTIME_ISSUE):
3679
					self.htmlFile.write('        <td bgcolor = "lightcoral" >KO - ' + machine + ' process faced Real Time issue(s)</td>\n')
3680
				elif (processesStatus == ENB_PROCESS_NOLOGFILE_TO_ANALYZE) or (processesStatus == OAI_UE_PROCESS_NOLOGFILE_TO_ANALYZE):
3681
					self.htmlFile.write('        <td bgcolor = "orange" >OK?</td>\n')
3682 3683
				elif (processesStatus == ENB_PROCESS_SLAVE_RRU_NOT_SYNCED):
					self.htmlFile.write('        <td bgcolor = "lightcoral" >KO - ' + machine + ' Slave RRU could not synch</td>\n')
3684
				elif (processesStatus == OAI_UE_PROCESS_COULD_NOT_SYNC):
3685
					self.htmlFile.write('        <td bgcolor = "lightcoral" >KO - UE could not sync</td>\n')
3686
				elif (processesStatus == HSS_PROCESS_FAILED):
Raphael Defosseux's avatar
Raphael Defosseux committed
3687
					self.htmlFile.write('        <td bgcolor = "lightcoral" >KO - HSS process not found</td>\n')
3688
				elif (processesStatus == MME_PROCESS_FAILED):
Raphael Defosseux's avatar
Raphael Defosseux committed
3689
					self.htmlFile.write('        <td bgcolor = "lightcoral" >KO - MME process not found</td>\n')
3690
				elif (processesStatus == SPGW_PROCESS_FAILED):
Raphael Defosseux's avatar
Raphael Defosseux committed
3691
					self.htmlFile.write('        <td bgcolor = "lightcoral" >KO - SPGW process not found</td>\n')
3692 3693
				elif (processesStatus == UE_IP_ADDRESS_ISSUE):
					self.htmlFile.write('        <td bgcolor = "lightcoral" >KO - Could not retrieve UE IP address</td>\n')
Raphael Defosseux's avatar
Raphael Defosseux committed
3694 3695
				else:
					self.htmlFile.write('        <td bgcolor = "lightcoral" >' + str(status)  + '</td>\n')
Raphael Defosseux's avatar
Raphael Defosseux committed
3696 3697
			else:
				self.htmlFile.write('        <td bgcolor = "orange" >' + str(status)  + '</td>\n')
Raphael Defosseux's avatar
Raphael Defosseux committed
3698
			if (len(str(self.htmleNBFailureMsg)) > 2):
3699 3700 3701 3702 3703
				cellBgColor = 'white'
				result = re.search('ended with|faced real time issues', self.htmleNBFailureMsg)
				if result is not None:
					cellBgColor = 'red'
				else:
3704
					result = re.search('showed|Reestablishment|Could not copy eNB logfile', self.htmleNBFailureMsg)
3705 3706
					if result is not None:
						cellBgColor = 'orange'
3707
				self.htmlFile.write('        <td bgcolor = "' + cellBgColor + '" colspan=' + str(self.htmlUEConnected) + '><pre style="background-color:' + cellBgColor + '">' + self.htmleNBFailureMsg + '</pre></td>\n')
Raphael Defosseux's avatar
Raphael Defosseux committed
3708
				self.htmleNBFailureMsg = ''
3709 3710 3711 3712 3713 3714
			elif (len(str(self.htmlUEFailureMsg)) > 2):
				cellBgColor = 'white'
				result = re.search('ended with|faced real time issues', self.htmlUEFailureMsg)
				if result is not None:
					cellBgColor = 'red'
				else:
3715
					result = re.search('showed|Could not copy UE logfile|oaitun_ue1 interface is either NOT mounted or NOT configured', self.htmlUEFailureMsg)
3716 3717 3718 3719
					if result is not None:
						cellBgColor = 'orange'
				self.htmlFile.write('        <td bgcolor = "' + cellBgColor + '" colspan=' + str(self.htmlUEConnected) + '><pre style="background-color:' + cellBgColor + '">' + self.htmlUEFailureMsg + '</pre></td>\n')
				self.htmlUEFailureMsg = ''
3720 3721 3722 3723 3724
			else:
				i = 0
				while (i < self.htmlUEConnected):
					self.htmlFile.write('        <td>-</td>\n')
					i += 1
Raphael Defosseux's avatar
Raphael Defosseux committed
3725 3726
			self.htmlFile.write('      </tr>\n')

Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
3727 3728
	def CreateHtmlTestRowQueue(self, options, status, ue_status, ue_queue):
		if ((not self.htmlFooterCreated) and (self.htmlHeaderCreated)):
Raphael Defosseux's avatar
Raphael Defosseux committed
3729
			addOrangeBK = False
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
3730
			self.htmlFile.write('      <tr>\n')
3731 3732
			self.htmlFile.write('        <td bgcolor = "lightcyan" >' + self.testCase_id  + '</td>\n')
			self.htmlFile.write('        <td>' + self.desc  + '</td>\n')
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
3733 3734 3735 3736 3737 3738
			self.htmlFile.write('        <td>' + str(options)  + '</td>\n')
			if (str(status) == 'OK'):
				self.htmlFile.write('        <td bgcolor = "lightgreen" >' + str(status)  + '</td>\n')
			elif (str(status) == 'KO'):
				self.htmlFile.write('        <td bgcolor = "lightcoral" >' + str(status)  + '</td>\n')
			else:
Raphael Defosseux's avatar
Raphael Defosseux committed
3739
				addOrangeBK = True
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
3740 3741 3742 3743 3744
				self.htmlFile.write('        <td bgcolor = "orange" >' + str(status)  + '</td>\n')
			i = 0
			while (i < self.htmlUEConnected):
				if (i < ue_status):
					if (not ue_queue.empty()):
Raphael Defosseux's avatar
Raphael Defosseux committed
3745
						if (addOrangeBK):
3746
							self.htmlFile.write('        <td bgcolor = "orange" >' + str(ue_queue.get()).replace('white', 'orange') + '</td>\n')
Raphael Defosseux's avatar
Raphael Defosseux committed
3747 3748
						else:
							self.htmlFile.write('        <td>' + str(ue_queue.get()) + '</td>\n')
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
3749 3750 3751 3752 3753 3754 3755
					else:
						self.htmlFile.write('        <td>-</td>\n')
				else:
					self.htmlFile.write('        <td>-</td>\n')
				i += 1
			self.htmlFile.write('      </tr>\n')

3756 3757 3758 3759 3760 3761 3762 3763 3764
#-----------------------------------------------------------
# ShowTestID()
#-----------------------------------------------------------
	def ShowTestID(self):
		logging.debug('\u001B[1m----------------------------------------\u001B[0m')
		logging.debug('\u001B[1mTest ID:' + self.testCase_id + '\u001B[0m')
		logging.debug('\u001B[1m' + self.desc + '\u001B[0m')
		logging.debug('\u001B[1m----------------------------------------\u001B[0m')

Raphael Defosseux's avatar
Raphael Defosseux committed
3765
#-----------------------------------------------------------
3766 3767 3768 3769 3770 3771 3772 3773 3774 3775
# Usage()
#-----------------------------------------------------------
def Usage():
	print('------------------------------------------------------------')
	print('main.py Ver:' + Version)
	print('------------------------------------------------------------')
	print('Usage: python main.py [options]')
	print('  --help  Show this help.')
	print('  --mode=[Mode]')
	print('      TesteNB')
3776
	print('      InitiateHtml, FinalizeHtml')
3777 3778
	print('      TerminateeNB, TerminateUE, TerminateHSS, TerminateMME, TerminateSPGW')
	print('      LogCollectBuild, LogCollecteNB, LogCollectHSS, LogCollectMME, LogCollectSPGW, LogCollectPing, LogCollectIperf')
3779 3780 3781 3782 3783
	print('  --eNBRepository=[eNB\'s Repository URL]                             or --ranRepository=[OAI RAN Repository URL]')
	print('  --eNBBranch=[eNB\'s Branch Name]                                    or --ranBranch=[OAI RAN Repository Branch')
	print('  --eNBCommitID=[eNB\'s Commit Number]                                or --ranCommitID=[OAI RAN Repository Commit SHA-1')
	print('  --eNB_AllowMerge=[eNB\'s Allow Merge Request (with target branch)]  or --ranAllowMerge=true/false')
	print('  --eNBTargetBranch=[eNB\'s Target Branch in case of a Merge Request] or --ranTargetBranch=[Target Branch]')
3784 3785 3786 3787 3788 3789 3790 3791 3792 3793 3794 3795
	print('  --eNBIPAddress=[eNB\'s IP Address]')
	print('  --eNBUserName=[eNB\'s Login User Name]')
	print('  --eNBPassword=[eNB\'s Login Password]')
	print('  --eNBSourceCodePath=[eNB\'s Source Code Path]')
	print('  --EPCIPAddress=[EPC\'s IP Address]')
	print('  --EPCUserName=[EPC\'s Login User Name]')
	print('  --EPCPassword=[EPC\'s Login Password]')
	print('  --EPCSourceCodePath=[EPC\'s Source Code Path]')
	print('  --EPCType=[EPC\'s Type: OAI or ltebox]')
	print('  --ADBIPAddress=[ADB\'s IP Address]')
	print('  --ADBUserName=[ADB\'s Login User Name]')
	print('  --ADBPassword=[ADB\'s Login Password]')
3796
	print('  --XMLTestFile=[XML Test File to be run]')
3797 3798 3799
	print('------------------------------------------------------------')

def CheckClassValidity(action,id):
3800
	if action != 'Build_eNB' and action != 'WaitEndBuild_eNB' and action != 'Initialize_eNB' and action != 'Terminate_eNB' and action != 'Initialize_UE' and action != 'Terminate_UE' and action != 'Attach_UE' and action != 'Detach_UE' and action != 'Build_OAI_UE' and action != 'Initialize_OAI_UE' and action != 'Terminate_OAI_UE' and action != 'DataDisable_UE' and action != 'DataEnable_UE' and action != 'CheckStatusUE' and action != 'Ping' and action != 'Iperf' and action != 'Reboot_UE' and action != 'Initialize_FlexranCtrl' and action != 'Terminate_FlexranCtrl' and action != 'Initialize_HSS' and action != 'Terminate_HSS' and action != 'Initialize_MME' and action != 'Terminate_MME' and action != 'Initialize_SPGW' and action != 'Terminate_SPGW' and action != 'Initialize_CatM_module' and action != 'Terminate_CatM_module' and action != 'Attach_CatM_module' and action != 'Detach_CatM_module' and action != 'Ping_CatM_module' and action != 'IdleSleep':
3801 3802 3803 3804 3805 3806 3807
		logging.debug('ERROR: test-case ' + id + ' has wrong class ' + action)
		return False
	return True

def GetParametersFromXML(action):
	if action == 'Build_eNB':
		SSH.Build_eNB_args = test.findtext('Build_eNB_args')
3808 3809 3810 3811 3812 3813
		SSH.eNB_instance = test.findtext('eNB_instance')
		if (SSH.eNB_instance is None):
			SSH.eNB_instance = '0'
		SSH.eNB_serverId = test.findtext('eNB_serverId')
		if (SSH.eNB_serverId is None):
			SSH.eNB_serverId = '0'
3814 3815 3816 3817 3818 3819 3820 3821 3822 3823 3824 3825 3826 3827 3828 3829 3830
		xmlBgBuildField = test.findtext('backgroundBuild')
		if (xmlBgBuildField is None):
			SSH.backgroundBuild = False
		else:
			if re.match('true', xmlBgBuildField, re.IGNORECASE):
				SSH.backgroundBuild = True
			else:
				SSH.backgroundBuild = False

	if action == 'WaitEndBuild_eNB':
		SSH.Build_eNB_args = test.findtext('Build_eNB_args')
		SSH.eNB_instance = test.findtext('eNB_instance')
		if (SSH.eNB_instance is None):
			SSH.eNB_instance = '0'
		SSH.eNB_serverId = test.findtext('eNB_serverId')
		if (SSH.eNB_serverId is None):
			SSH.eNB_serverId = '0'
3831

3832 3833
	if action == 'Initialize_eNB':
		SSH.Initialize_eNB_args = test.findtext('Initialize_eNB_args')
3834 3835 3836
		SSH.eNB_instance = test.findtext('eNB_instance')
		if (SSH.eNB_instance is None):
			SSH.eNB_instance = '0'
3837 3838 3839
		SSH.eNB_serverId = test.findtext('eNB_serverId')
		if (SSH.eNB_serverId is None):
			SSH.eNB_serverId = '0'
3840 3841 3842 3843 3844

	if action == 'Terminate_eNB':
		SSH.eNB_instance = test.findtext('eNB_instance')
		if (SSH.eNB_instance is None):
			SSH.eNB_instance = '0'
3845 3846 3847
		SSH.eNB_serverId = test.findtext('eNB_serverId')
		if (SSH.eNB_serverId is None):
			SSH.eNB_serverId = '0'
3848

3849 3850 3851 3852 3853 3854 3855
	if action == 'Attach_UE':
		nbMaxUEtoAttach = test.findtext('nbMaxUEtoAttach')
		if (nbMaxUEtoAttach is None):
			SSH.nbMaxUEtoAttach = -1
		else:
			SSH.nbMaxUEtoAttach = int(nbMaxUEtoAttach)

3856 3857 3858 3859 3860 3861 3862
	if action == 'CheckStatusUE':
		expectedNBUE = test.findtext('expectedNbOfConnectedUEs')
		if (expectedNBUE is None):
			SSH.expectedNbOfConnectedUEs = -1
		else:
			SSH.expectedNbOfConnectedUEs = int(expectedNBUE)

Boris Djalal's avatar
Boris Djalal committed
3863 3864 3865 3866 3867 3868 3869 3870 3871 3872 3873 3874 3875 3876
	if action == 'Build_OAI_UE':
		SSH.Build_OAI_UE_args = test.findtext('Build_OAI_UE_args')

	if action == 'Initialize_OAI_UE':
		SSH.Initialize_OAI_UE_args = test.findtext('Initialize_OAI_UE_args')
		SSH.UE_instance = test.findtext('UE_instance')
		if (SSH.UE_instance is None):
			SSH.UE_instance = '0'

	if action == 'Terminate_OAI_UE':
		SSH.eNB_instance = test.findtext('UE_instance')
		if (SSH.UE_instance is None):
			SSH.UE_instance = '0'

3877
	if action == 'Ping' or action == 'Ping_CatM_module':
3878 3879 3880 3881 3882 3883
		SSH.ping_args = test.findtext('ping_args')
		SSH.ping_packetloss_threshold = test.findtext('ping_packetloss_threshold')

	if action == 'Iperf':
		SSH.iperf_args = test.findtext('iperf_args')
		SSH.iperf_packetloss_threshold = test.findtext('iperf_packetloss_threshold')
3884 3885 3886 3887 3888 3889 3890
		SSH.iperf_profile = test.findtext('iperf_profile')
		if (SSH.iperf_profile is None):
			SSH.iperf_profile = 'balanced'
		else:
			if SSH.iperf_profile != 'balanced' and SSH.iperf_profile != 'unbalanced' and SSH.iperf_profile != 'single-ue':
				logging.debug('ERROR: test-case has wrong profile ' + SSH.iperf_profile)
				SSH.iperf_profile = 'balanced'
3891

3892 3893 3894 3895 3896 3897 3898
	if action == 'IdleSleep':
		string_field = test.findtext('idle_sleep_time_in_sec')
		if (string_field is None):
			SSH.idle_sleep_time = 5
		else:
			SSH.idle_sleep_time = int(string_field)

3899 3900 3901 3902 3903 3904 3905 3906 3907 3908 3909 3910 3911 3912 3913 3914 3915 3916 3917 3918
#check if given test is in list
#it is in list if one of the strings in 'list' is at the beginning of 'test'
def test_in_list(test, list):
	for check in list:
		check=check.replace('+','')
		if (test.startswith(check)):
			return True
	return False

def receive_signal(signum, frame):
	sys.exit(1)

#-----------------------------------------------------------
# Parameter Check
#-----------------------------------------------------------
mode = ''
SSH = SSHConnection()

argvs = sys.argv
argc = len(argvs)
3919
cwd = os.getcwd()
3920 3921 3922 3923 3924 3925 3926 3927 3928

while len(argvs) > 1:
	myArgv = argvs.pop(1)	# 0th is this file's name
	if re.match('^\-\-help$', myArgv, re.IGNORECASE):
		Usage()
		sys.exit(0)
	elif re.match('^\-\-mode=(.+)$', myArgv, re.IGNORECASE):
		matchReg = re.match('^\-\-mode=(.+)$', myArgv, re.IGNORECASE)
		mode = matchReg.group(1)
3929 3930 3931 3932 3933 3934 3935 3936 3937 3938 3939
	elif re.match('^\-\-eNBRepository=(.+)$|^\-\-ranRepository(.+)$', myArgv, re.IGNORECASE):
		if re.match('^\-\-eNBRepository=(.+)$', myArgv, re.IGNORECASE):
			matchReg = re.match('^\-\-eNBRepository=(.+)$', myArgv, re.IGNORECASE)
		else:
			matchReg = re.match('^\-\-ranRepository=(.+)$', myArgv, re.IGNORECASE)
		SSH.ranRepository = matchReg.group(1)
	elif re.match('^\-\-eNB_AllowMerge=(.+)$|^\-\-ranAllowMerge=(.+)$', myArgv, re.IGNORECASE):
		if re.match('^\-\-eNB_AllowMerge=(.+)$', myArgv, re.IGNORECASE):
			matchReg = re.match('^\-\-eNB_AllowMerge=(.+)$', myArgv, re.IGNORECASE)
		else:
			matchReg = re.match('^\-\-ranAllowMerge=(.+)$', myArgv, re.IGNORECASE)
Raphael Defosseux's avatar
Raphael Defosseux committed
3940 3941
		doMerge = matchReg.group(1)
		if ((doMerge == 'true') or (doMerge == 'True')):
3942 3943 3944 3945 3946 3947 3948 3949 3950 3951 3952 3953 3954 3955 3956 3957 3958 3959 3960 3961 3962 3963 3964 3965 3966 3967 3968 3969 3970 3971 3972 3973 3974 3975 3976 3977 3978 3979 3980 3981 3982 3983 3984 3985 3986 3987 3988 3989 3990 3991 3992 3993 3994 3995 3996 3997 3998 3999 4000
			SSH.ranAllowMerge = True
	elif re.match('^\-\-eNBBranch=(.+)$|^\-\-ranBranch=(.+)$', myArgv, re.IGNORECASE):
		if re.match('^\-\-eNBBranch=(.+)$', myArgv, re.IGNORECASE):
			matchReg = re.match('^\-\-eNBBranch=(.+)$', myArgv, re.IGNORECASE)
		else:
			matchReg = re.match('^\-\-ranBranch=(.+)$', myArgv, re.IGNORECASE)
		SSH.ranBranch = matchReg.group(1)
	elif re.match('^\-\-eNBCommitID=(.*)$|^\-\-ranCommitID=(.*)$', myArgv, re.IGNORECASE):
		if re.match('^\-\-eNBCommitID=(.*)$', myArgv, re.IGNORECASE):
			matchReg = re.match('^\-\-eNBCommitID=(.*)$', myArgv, re.IGNORECASE)
		else:
			matchReg = re.match('^\-\-ranCommitID=(.*)$', myArgv, re.IGNORECASE)
		SSH.ranCommitID = matchReg.group(1)
	elif re.match('^\-\-eNBTargetBranch=(.*)$|^\-\-ranTargetBranch=(.*)$', myArgv, re.IGNORECASE):
		if re.match('^\-\-eNBTargetBranch=(.*)$', myArgv, re.IGNORECASE):
			matchReg = re.match('^\-\-eNBTargetBranch=(.*)$', myArgv, re.IGNORECASE)
		else:
			matchReg = re.match('^\-\-ranTargetBranch=(.*)$', myArgv, re.IGNORECASE)
		SSH.ranTargetBranch = matchReg.group(1)
	elif re.match('^\-\-eNBIPAddress=(.+)$|^\-\-eNB[1-2]IPAddress=(.+)$', myArgv, re.IGNORECASE):
		if re.match('^\-\-eNBIPAddress=(.+)$', myArgv, re.IGNORECASE):
			matchReg = re.match('^\-\-eNBIPAddress=(.+)$', myArgv, re.IGNORECASE)
			SSH.eNBIPAddress = matchReg.group(1)
		elif re.match('^\-\-eNB1IPAddress=(.+)$', myArgv, re.IGNORECASE):
			matchReg = re.match('^\-\-eNB1IPAddress=(.+)$', myArgv, re.IGNORECASE)
			SSH.eNB1IPAddress = matchReg.group(1)
		elif re.match('^\-\-eNB2IPAddress=(.+)$', myArgv, re.IGNORECASE):
			matchReg = re.match('^\-\-eNB2IPAddress=(.+)$', myArgv, re.IGNORECASE)
			SSH.eNB2IPAddress = matchReg.group(1)
	elif re.match('^\-\-eNBUserName=(.+)$|^\-\-eNB[1-2]UserName=(.+)$', myArgv, re.IGNORECASE):
		if re.match('^\-\-eNBUserName=(.+)$', myArgv, re.IGNORECASE):
			matchReg = re.match('^\-\-eNBUserName=(.+)$', myArgv, re.IGNORECASE)
			SSH.eNBUserName = matchReg.group(1)
		elif re.match('^\-\-eNB1UserName=(.+)$', myArgv, re.IGNORECASE):
			matchReg = re.match('^\-\-eNB1UserName=(.+)$', myArgv, re.IGNORECASE)
			SSH.eNB1UserName = matchReg.group(1)
		elif re.match('^\-\-eNB2UserName=(.+)$', myArgv, re.IGNORECASE):
			matchReg = re.match('^\-\-eNB2UserName=(.+)$', myArgv, re.IGNORECASE)
			SSH.eNB2UserName = matchReg.group(1)
	elif re.match('^\-\-eNBPassword=(.+)$|^\-\-eNB[1-2]Password=(.+)$', myArgv, re.IGNORECASE):
		if re.match('^\-\-eNBPassword=(.+)$', myArgv, re.IGNORECASE):
			matchReg = re.match('^\-\-eNBPassword=(.+)$', myArgv, re.IGNORECASE)
			SSH.eNBPassword = matchReg.group(1)
		elif re.match('^\-\-eNB1Password=(.+)$', myArgv, re.IGNORECASE):
			matchReg = re.match('^\-\-eNB1Password=(.+)$', myArgv, re.IGNORECASE)
			SSH.eNB1Password = matchReg.group(1)
		elif re.match('^\-\-eNB2Password=(.+)$', myArgv, re.IGNORECASE):
			matchReg = re.match('^\-\-eNB2Password=(.+)$', myArgv, re.IGNORECASE)
			SSH.eNB2Password = matchReg.group(1)
	elif re.match('^\-\-eNBSourceCodePath=(.+)$|^\-\-eNB[1-2]SourceCodePath=(.+)$', myArgv, re.IGNORECASE):
		if re.match('^\-\-eNBSourceCodePath=(.+)$', myArgv, re.IGNORECASE):
			matchReg = re.match('^\-\-eNBSourceCodePath=(.+)$', myArgv, re.IGNORECASE)
			SSH.eNBSourceCodePath = matchReg.group(1)
		elif re.match('^\-\-eNB1SourceCodePath=(.+)$', myArgv, re.IGNORECASE):
			matchReg = re.match('^\-\-eNB1SourceCodePath=(.+)$', myArgv, re.IGNORECASE)
			SSH.eNB1SourceCodePath = matchReg.group(1)
		elif re.match('^\-\-eNB2SourceCodePath=(.+)$', myArgv, re.IGNORECASE):
			matchReg = re.match('^\-\-eNB2SourceCodePath=(.+)$', myArgv, re.IGNORECASE)
			SSH.eNB2SourceCodePath = matchReg.group(1)
4001 4002 4003 4004 4005 4006 4007 4008 4009 4010 4011 4012 4013 4014 4015 4016 4017 4018 4019 4020 4021 4022 4023 4024 4025 4026 4027 4028 4029 4030
	elif re.match('^\-\-EPCIPAddress=(.+)$', myArgv, re.IGNORECASE):
		matchReg = re.match('^\-\-EPCIPAddress=(.+)$', myArgv, re.IGNORECASE)
		SSH.EPCIPAddress = matchReg.group(1)
	elif re.match('^\-\-EPCBranch=(.+)$', myArgv, re.IGNORECASE):
		matchReg = re.match('^\-\-EPCBranch=(.+)$', myArgv, re.IGNORECASE)
		SSH.EPCBranch = matchReg.group(1)
	elif re.match('^\-\-EPCUserName=(.+)$', myArgv, re.IGNORECASE):
		matchReg = re.match('^\-\-EPCUserName=(.+)$', myArgv, re.IGNORECASE)
		SSH.EPCUserName = matchReg.group(1)
	elif re.match('^\-\-EPCPassword=(.+)$', myArgv, re.IGNORECASE):
		matchReg = re.match('^\-\-EPCPassword=(.+)$', myArgv, re.IGNORECASE)
		SSH.EPCPassword = matchReg.group(1)
	elif re.match('^\-\-EPCSourceCodePath=(.+)$', myArgv, re.IGNORECASE):
		matchReg = re.match('^\-\-EPCSourceCodePath=(.+)$', myArgv, re.IGNORECASE)
		SSH.EPCSourceCodePath = matchReg.group(1)
	elif re.match('^\-\-EPCType=(.+)$', myArgv, re.IGNORECASE):
		matchReg = re.match('^\-\-EPCType=(.+)$', myArgv, re.IGNORECASE)
		if re.match('OAI', matchReg.group(1), re.IGNORECASE) or re.match('ltebox', matchReg.group(1), re.IGNORECASE):
			SSH.EPCType = matchReg.group(1)
		else:
			sys.exit('Invalid EPC Type: ' + matchReg.group(1) + ' -- (should be OAI or ltebox)')
	elif re.match('^\-\-ADBIPAddress=(.+)$', myArgv, re.IGNORECASE):
		matchReg = re.match('^\-\-ADBIPAddress=(.+)$', myArgv, re.IGNORECASE)
		SSH.ADBIPAddress = matchReg.group(1)
	elif re.match('^\-\-ADBUserName=(.+)$', myArgv, re.IGNORECASE):
		matchReg = re.match('^\-\-ADBUserName=(.+)$', myArgv, re.IGNORECASE)
		SSH.ADBUserName = matchReg.group(1)
	elif re.match('^\-\-ADBPassword=(.+)$', myArgv, re.IGNORECASE):
		matchReg = re.match('^\-\-ADBPassword=(.+)$', myArgv, re.IGNORECASE)
		SSH.ADBPassword = matchReg.group(1)
4031 4032
	elif re.match('^\-\-XMLTestFile=(.+)$', myArgv, re.IGNORECASE):
		matchReg = re.match('^\-\-XMLTestFile=(.+)$', myArgv, re.IGNORECASE)
4033 4034
		SSH.testXMLfiles.append(matchReg.group(1))
		SSH.nbTestXMLfiles += 1
4035 4036
	elif re.match('^\-\-UEIPAddress=(.+)$', myArgv, re.IGNORECASE):
		matchReg = re.match('^\-\-UEIPAddress=(.+)$', myArgv, re.IGNORECASE)
4037
		SSH.UEIPAddress = matchReg.group(1)
4038 4039 4040 4041 4042 4043
	elif re.match('^\-\-UEUserName=(.+)$', myArgv, re.IGNORECASE):
		matchReg = re.match('^\-\-UEUserName=(.+)$', myArgv, re.IGNORECASE)
		SSH.UEUserName = matchReg.group(1)
	elif re.match('^\-\-UEPassword=(.+)$', myArgv, re.IGNORECASE):
		matchReg = re.match('^\-\-UEPassword=(.+)$', myArgv, re.IGNORECASE)
		SSH.UEPassword = matchReg.group(1)
4044 4045 4046
	elif re.match('^\-\-UESourceCodePath=(.+)$', myArgv, re.IGNORECASE):
		matchReg = re.match('^\-\-UESourceCodePath=(.+)$', myArgv, re.IGNORECASE)
		SSH.UESourceCodePath = matchReg.group(1)
4047 4048 4049 4050 4051
	elif re.match('^\-\-finalStatus=(.+)$', myArgv, re.IGNORECASE):
		matchReg = re.match('^\-\-finalStatus=(.+)$', myArgv, re.IGNORECASE)
		finalStatus = matchReg.group(1)
		if ((finalStatus == 'true') or (finalStatus == 'True')):
			SSH.finalStatus = True
4052 4053 4054 4055 4056 4057 4058 4059
	else:
		Usage()
		sys.exit('Invalid Parameter: ' + myArgv)

if re.match('^TerminateeNB$', mode, re.IGNORECASE):
	if SSH.eNBIPAddress == '' or SSH.eNBUserName == '' or SSH.eNBPassword == '':
		Usage()
		sys.exit('Insufficient Parameter')
Raphael Defosseux's avatar
Raphael Defosseux committed
4060
	SSH.eNB_serverId = '0'
4061 4062
	SSH.eNB_instance = '0'
	SSH.eNBSourceCodePath = '/tmp/'
4063 4064
	SSH.TerminateeNB()
elif re.match('^TerminateUE$', mode, re.IGNORECASE):
4065
	if (SSH.ADBIPAddress == '' or SSH.ADBUserName == '' or SSH.ADBPassword == ''):
4066 4067 4068 4069
		Usage()
		sys.exit('Insufficient Parameter')
	signal.signal(signal.SIGUSR1, receive_signal)
	SSH.TerminateUE()
Boris Djalal's avatar
Boris Djalal committed
4070 4071 4072 4073 4074
elif re.match('^TerminateOAIUE$', mode, re.IGNORECASE):
	if SSH.UEIPAddress == '' or SSH.UEUserName == '' or SSH.UEPassword == '':
		Usage()
		sys.exit('Insufficient Parameter')
	signal.signal(signal.SIGUSR1, receive_signal)
4075
	SSH.TerminateOAIUE()
4076
elif re.match('^TerminateHSS$', mode, re.IGNORECASE):
4077
	if SSH.EPCIPAddress == '' or SSH.EPCUserName == '' or SSH.EPCPassword == '' or SSH.EPCType == '' or SSH.EPCSourceCodePath == '':
4078 4079 4080 4081
		Usage()
		sys.exit('Insufficient Parameter')
	SSH.TerminateHSS()
elif re.match('^TerminateMME$', mode, re.IGNORECASE):
4082
	if SSH.EPCIPAddress == '' or SSH.EPCUserName == '' or SSH.EPCPassword == '' or SSH.EPCType == '' or SSH.EPCSourceCodePath == '':
4083 4084 4085 4086
		Usage()
		sys.exit('Insufficient Parameter')
	SSH.TerminateMME()
elif re.match('^TerminateSPGW$', mode, re.IGNORECASE):
4087
	if SSH.EPCIPAddress == '' or SSH.EPCUserName == '' or SSH.EPCPassword == '' or SSH.EPCType == '' or SSH.EPCSourceCodePath == '':
4088 4089 4090 4091
		Usage()
		sys.exit('Insufficient Parameter')
	SSH.TerminateSPGW()
elif re.match('^LogCollectBuild$', mode, re.IGNORECASE):
Boris Djalal's avatar
Boris Djalal committed
4092
	if (SSH.eNBIPAddress == '' or SSH.eNBUserName == '' or SSH.eNBPassword == '' or SSH.eNBSourceCodePath == '') and (SSH.UEIPAddress == '' or SSH.UEUserName == '' or SSH.UEPassword == '' or SSH.UESourceCodePath == ''):
4093 4094 4095 4096 4097 4098 4099 4100 4101
		Usage()
		sys.exit('Insufficient Parameter')
	SSH.LogCollectBuild()
elif re.match('^LogCollecteNB$', mode, re.IGNORECASE):
	if SSH.eNBIPAddress == '' or SSH.eNBUserName == '' or SSH.eNBPassword == '' or SSH.eNBSourceCodePath == '':
		Usage()
		sys.exit('Insufficient Parameter')
	SSH.LogCollecteNB()
elif re.match('^LogCollectHSS$', mode, re.IGNORECASE):
4102
	if SSH.EPCIPAddress == '' or SSH.EPCUserName == '' or SSH.EPCPassword == '' or SSH.EPCType == '' or SSH.EPCSourceCodePath == '':
4103 4104 4105 4106
		Usage()
		sys.exit('Insufficient Parameter')
	SSH.LogCollectHSS()
elif re.match('^LogCollectMME$', mode, re.IGNORECASE):
4107
	if SSH.EPCIPAddress == '' or SSH.EPCUserName == '' or SSH.EPCPassword == '' or SSH.EPCType == '' or SSH.EPCSourceCodePath == '':
4108 4109 4110 4111
		Usage()
		sys.exit('Insufficient Parameter')
	SSH.LogCollectMME()
elif re.match('^LogCollectSPGW$', mode, re.IGNORECASE):
4112
	if SSH.EPCIPAddress == '' or SSH.EPCUserName == '' or SSH.EPCPassword == '' or SSH.EPCType == '' or SSH.EPCSourceCodePath == '':
4113 4114 4115 4116 4117 4118 4119 4120 4121 4122 4123 4124 4125
		Usage()
		sys.exit('Insufficient Parameter')
	SSH.LogCollectSPGW()
elif re.match('^LogCollectPing$', mode, re.IGNORECASE):
	if SSH.EPCIPAddress == '' or SSH.EPCUserName == '' or SSH.EPCPassword == '' or SSH.EPCSourceCodePath == '':
		Usage()
		sys.exit('Insufficient Parameter')
	SSH.LogCollectPing()
elif re.match('^LogCollectIperf$', mode, re.IGNORECASE):
	if SSH.EPCIPAddress == '' or SSH.EPCUserName == '' or SSH.EPCPassword == '' or SSH.EPCSourceCodePath == '':
		Usage()
		sys.exit('Insufficient Parameter')
	SSH.LogCollectIperf()
Boris Djalal's avatar
Boris Djalal committed
4126
elif re.match('^LogCollectOAIUE$', mode, re.IGNORECASE):
4127 4128 4129 4130
	if SSH.UEIPAddress == '' or SSH.UEUserName == '' or SSH.UEPassword == '' or SSH.UESourceCodePath == '':
		Usage()
		sys.exit('Insufficient Parameter')
	SSH.LogCollectOAIUE()
4131
elif re.match('^InitiateHtml$', mode, re.IGNORECASE):
Boris Djalal's avatar
Boris Djalal committed
4132
	if (SSH.ADBIPAddress == '' or SSH.ADBUserName == '' or SSH.ADBPassword == ''):
4133 4134 4135
		Usage()
		sys.exit('Insufficient Parameter')
	count = 0
4136
	foundCount = 0
4137
	while (count < SSH.nbTestXMLfiles):
4138
		xml_test_file = cwd + "/" + SSH.testXMLfiles[count]
4139 4140 4141 4142 4143 4144 4145 4146
		xml_test_file = sys.path[0] + "/" + SSH.testXMLfiles[count]
		if (os.path.isfile(xml_test_file)):
			xmlTree = ET.parse(xml_test_file)
			xmlRoot = xmlTree.getroot()
			SSH.htmlTabRefs.append(xmlRoot.findtext('htmlTabRef',default='test-tab-' + str(count)))
			SSH.htmlTabNames.append(xmlRoot.findtext('htmlTabName',default='Test-' + str(count)))
			SSH.htmlTabIcons.append(xmlRoot.findtext('htmlTabIcon',default='info-sign'))
			foundCount += 1
4147
		count += 1
4148 4149
	if foundCount != SSH.nbTestXMLfiles:
		SSH.nbTestXMLfiles = foundCount
4150 4151 4152
	SSH.CreateHtmlHeader()
elif re.match('^FinalizeHtml$', mode, re.IGNORECASE):
	SSH.CreateHtmlFooter(SSH.finalStatus)
4153 4154
elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re.IGNORECASE):
	if re.match('^TesteNB$', mode, re.IGNORECASE):
4155
		if SSH.eNBIPAddress == '' or SSH.ranRepository == '' or SSH.ranBranch == '' or SSH.eNBUserName == '' or SSH.eNBPassword == '' or SSH.eNBSourceCodePath == '' or SSH.EPCIPAddress == '' or SSH.EPCUserName == '' or SSH.EPCPassword == '' or SSH.EPCType == '' or SSH.EPCSourceCodePath == '' or SSH.ADBIPAddress == '' or SSH.ADBUserName == '' or SSH.ADBPassword == '':
4156 4157 4158
			Usage()
			sys.exit('Insufficient Parameter')

4159
		if (SSH.EPCIPAddress != '') and (SSH.EPCIPAddress != 'none'):
4160 4161
			SSH.copyout(SSH.EPCIPAddress, SSH.EPCUserName, SSH.EPCPassword, cwd + "/tcp_iperf_stats.awk", "/tmp")
			SSH.copyout(SSH.EPCIPAddress, SSH.EPCUserName, SSH.EPCPassword, cwd + "/active_net_interfaces.awk", "/tmp")
4162
	else:
4163
		if SSH.UEIPAddress == '' or SSH.ranRepository == '' or SSH.ranBranch == '' or SSH.UEUserName == '' or SSH.UEPassword == '' or SSH.UESourceCodePath == '':
4164 4165
			Usage()
			sys.exit('UE: Insufficient Parameter')
4166

4167
	#read test_case_list.xml file
4168
	# if no parameters for XML file, use default value
4169
	if (SSH.nbTestXMLfiles != 1):
4170
		xml_test_file = cwd + "/test_case_list.xml"
4171
	else:
4172
		xml_test_file = cwd + "/" + SSH.testXMLfiles[0]
4173 4174 4175 4176 4177 4178

	xmlTree = ET.parse(xml_test_file)
	xmlRoot = xmlTree.getroot()

	exclusion_tests=xmlRoot.findtext('TestCaseExclusionList',default='')
	requested_tests=xmlRoot.findtext('TestCaseRequestedList',default='')
4179 4180
	if (SSH.nbTestXMLfiles == 1):
		SSH.htmlTabRefs.append(xmlRoot.findtext('htmlTabRef',default='test-tab-0'))
4181 4182 4183 4184 4185 4186 4187 4188 4189 4190 4191 4192 4193 4194 4195 4196 4197 4198 4199 4200 4201 4202 4203 4204 4205
	all_tests=xmlRoot.findall('testCase')

	exclusion_tests=exclusion_tests.split()
	requested_tests=requested_tests.split()

	#check that exclusion tests are well formatted
	#(6 digits or less than 6 digits followed by +)
	for test in exclusion_tests:
		if     (not re.match('^[0-9]{6}$', test) and
				not re.match('^[0-9]{1,5}\+$', test)):
			logging.debug('ERROR: exclusion test is invalidly formatted: ' + test)
			sys.exit(1)
		else:
			logging.debug(test)

	#check that requested tests are well formatted
	#(6 digits or less than 6 digits followed by +)
	#be verbose
	for test in requested_tests:
		if     (re.match('^[0-9]{6}$', test) or
				re.match('^[0-9]{1,5}\+$', test)):
			logging.debug('INFO: test group/case requested: ' + test)
		else:
			logging.debug('ERROR: requested test is invalidly formatted: ' + test)
			sys.exit(1)
4206 4207
	if (SSH.EPCIPAddress != '') and (SSH.EPCIPAddress != 'none'):
		SSH.CheckFlexranCtrlInstallation()
4208 4209 4210 4211 4212 4213 4214

	#get the list of tests to be done
	todo_tests=[]
	for test in requested_tests:
		if    (test_in_list(test, exclusion_tests)):
			logging.debug('INFO: test will be skipped: ' + test)
		else:
4215
			#logging.debug('INFO: test will be run: ' + test)
4216
			todo_tests.append(test)
4217

4218
	signal.signal(signal.SIGUSR1, receive_signal)
4219

4220 4221
	SSH.CreateHtmlTabHeader()

4222 4223 4224 4225 4226 4227 4228 4229 4230 4231
	for test_case_id in todo_tests:
		for test in all_tests:
			id = test.get('id')
			if test_case_id != id:
				continue
			SSH.testCase_id = id
			SSH.desc = test.findtext('desc')
			action = test.findtext('class')
			if (CheckClassValidity(action, id) == False):
				continue
4232
			SSH.ShowTestID()
4233
			GetParametersFromXML(action)
4234
			if action == 'Initialize_UE' or action == 'Attach_UE' or action == 'Detach_UE' or action == 'Ping' or action == 'Iperf' or action == 'Reboot_UE' or action == 'DataDisable_UE' or action == 'DataEnable_UE' or action == 'CheckStatusUE':
4235 4236 4237
				if (SSH.ADBIPAddress != 'none'):
					terminate_ue_flag = False
					SSH.GetAllUEDevices(terminate_ue_flag)
4238 4239
			if action == 'Build_eNB':
				SSH.BuildeNB()
4240 4241
			elif action == 'WaitEndBuild_eNB':
				SSH.WaitBuildeNBisFinished()
4242 4243 4244 4245 4246 4247 4248 4249 4250 4251 4252 4253
			elif action == 'Initialize_eNB':
				SSH.InitializeeNB()
			elif action == 'Terminate_eNB':
				SSH.TerminateeNB()
			elif action == 'Initialize_UE':
				SSH.InitializeUE()
			elif action == 'Terminate_UE':
				SSH.TerminateUE()
			elif action == 'Attach_UE':
				SSH.AttachUE()
			elif action == 'Detach_UE':
				SSH.DetachUE()
4254 4255 4256 4257 4258 4259
			elif action == 'DataDisable_UE':
				SSH.DataDisableUE()
			elif action == 'DataEnable_UE':
				SSH.DataEnableUE()
			elif action == 'CheckStatusUE':
				SSH.CheckStatusUE()
Boris Djalal's avatar
Boris Djalal committed
4260 4261 4262 4263 4264 4265
			elif action == 'Build_OAI_UE':
				SSH.BuildOAIUE()
			elif action == 'Initialize_OAI_UE':
				SSH.InitializeOAIUE()
			elif action == 'Terminate_OAI_UE':
				SSH.TerminateOAIUE()
4266 4267 4268 4269 4270 4271 4272 4273
			elif action == 'Initialize_CatM_module':
				SSH.InitializeCatM()
			elif action == 'Terminate_CatM_module':
				SSH.TerminateCatM()
			elif action == 'Attach_CatM_module':
				SSH.AttachCatM()
			elif action == 'Detach_CatM_module':
				SSH.TerminateCatM()
4274 4275
			elif action == 'Ping_CatM_module':
				SSH.PingCatM()
4276 4277 4278 4279 4280 4281 4282 4283 4284 4285 4286 4287 4288 4289 4290 4291 4292 4293
			elif action == 'Ping':
				SSH.Ping()
			elif action == 'Iperf':
				SSH.Iperf()
			elif action == 'Reboot_UE':
				SSH.RebootUE()
			elif action == 'Initialize_HSS':
				SSH.InitializeHSS()
			elif action == 'Terminate_HSS':
				SSH.TerminateHSS()
			elif action == 'Initialize_MME':
				SSH.InitializeMME()
			elif action == 'Terminate_MME':
				SSH.TerminateMME()
			elif action == 'Initialize_SPGW':
				SSH.InitializeSPGW()
			elif action == 'Terminate_SPGW':
				SSH.TerminateSPGW()
4294 4295 4296 4297
			elif action == 'Initialize_FlexranCtrl':
				SSH.InitializeFlexranCtrl()
			elif action == 'Terminate_FlexranCtrl':
				SSH.TerminateFlexranCtrl()
4298 4299
			elif action == 'IdleSleep':
				SSH.IdleSleep()
4300 4301
			else:
				sys.exit('Invalid action')
Raphael Defosseux's avatar
Raphael Defosseux committed
4302

4303
	SSH.CreateHtmlTabFooter(True)
4304 4305 4306 4307
else:
	Usage()
	sys.exit('Invalid mode')
sys.exit(0)