[CI][Docker] able to deploy/undeploy a monolithic eNB with a compose file:

  -- The compose file still keeps a few template fields to be replaced
  -- initial scenario has a full UE attach / traffic test
Signed-off-by: Raphael Defosseux <raphael.defosseux@eurecom.fr>

ci-scripts/cls_containerize.py

@@ -75,9 +75,16 @@ class Containerize():
 		self.imageKind = ''
 		self.eNB_instance = 0
 		self.eNB_serverId = ['', '', '']
+		self.yamlPath = ['', '', '']
+		self.eNB_logFile = ['', '', '']
+
 		self.testCase_id = ''
+
+		self.flexranCtrlDeployed = False
+		self.flexranCtrlIpAddress = ''
 		self.htmlObj = None
 		self.epcObj = None
+		self.ranObj = None
 
 #-----------------------------------------------------------
 # Container management functions
@@ -102,7 +109,6 @@ class Containerize():
 			lUserName = self.eNB2UserName
 			lPassWord = self.eNB2Password
 			lSourcePath = self.eNB2SourceCodePath
-		logging.debug('lIpAddr = ' + lIpAddr + ' lUserName = ' + lUserName + ' lPassWord = ' + lPassWord + ' lSourcePath = ' + lSourcePath)
 		if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '':
 			HELP.GenericHelp(CONST.Version)
 			sys.exit('Insufficient Parameter')
@@ -258,3 +264,166 @@ class Containerize():
 		if self.htmlObj is not None:
 			self.htmlObj.CreateHtmlTestRow(self.imageKind, 'OK', CONST.ALL_PROCESSES_OK)		
 
+	def DeployObject(self):
+		if self.eNB_serverId[self.eNB_instance] == '0':
+			lIpAddr = self.eNBIPAddress
+			lUserName = self.eNBUserName
+			lPassWord = self.eNBPassword
+			lSourcePath = self.eNBSourceCodePath
+		elif self.eNB_serverId[self.eNB_instance] == '1':
+			lIpAddr = self.eNB1IPAddress
+			lUserName = self.eNB1UserName
+			lPassWord = self.eNB1Password
+			lSourcePath = self.eNB1SourceCodePath
+		elif self.eNB_serverId[self.eNB_instance] == '2':
+			lIpAddr = self.eNB2IPAddress
+			lUserName = self.eNB2UserName
+			lPassWord = self.eNB2Password
+			lSourcePath = self.eNB2SourceCodePath
+		if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '':
+			HELP.GenericHelp(CONST.Version)
+			sys.exit('Insufficient Parameter')
+		logging.debug('\u001B[1m Deploying OAI Object on server: ' + lIpAddr + '\u001B[0m')
+		mySSH = SSH.SSHConnection()
+		mySSH.open(lIpAddr, lUserName, lPassWord)
+		# Putting the CPUs in a good state, we do that only on a few servers
+		mySSH.command('hostname', '\$', 5)
+		result = re.search('obelix|asterix',  mySSH.getBefore())
+		if result is not None:
+			mySSH.command('if command -v cpupower &> /dev/null; then echo ' + lPassWord + ' | sudo -S cpupower idle-set -D 0; fi', '\$', 5)
+			time.sleep(5)
+		
+		mySSH.command('cd ' + lSourcePath + '/' + self.yamlPath[self.eNB_instance], '\$', 5)
+		mySSH.command('cp docker-compose.yml ci-docker-compose.yml', '\$', 5)
+		imageTag = 'develop'
+		if (self.ranAllowMerge):
+			imageTag = 'ci-temp'
+		mySSH.command('sed -i -e "s/image: oai-enb:latest/image: oai-enb:' + imageTag + '/" ci-docker-compose.yml', '\$', 2)
+		if self.epcObj is not None:
+			localMmeIpAddr = self.epcObj.MmeIPAddress
+			mySSH.command('sed -i -e "s/CI_MME_IP_ADDR/' + localMmeIpAddr + '/" ci-docker-compose.yml', '\$', 2)
+		if self.flexranCtrlDeployed:
+			mySSH.command('sed -i -e \'s/FLEXRAN_ENABLED:.*/FLEXRAN_ENABLED: "yes"/\' ci-docker-compose.yml', '\$', 2)
+			mySSH.command('sed -i -e "s/CI_FLEXRAN_CTL_IP_ADDR/' + self.flexranCtrlIpAddress + '/" ci-docker-compose.yml', '\$', 2)
+		else:
+			mySSH.command('sed -i -e "s/FLEXRAN_ENABLED:.*$/FLEXRAN_ENABLED: \"no\"/" ci-docker-compose.yml', '\$', 2)
+			mySSH.command('sed -i -e "s/CI_FLEXRAN_CTL_IP_ADDR/127.0.0.1/" ci-docker-compose.yml', '\$', 2)
+		# Currently support only one
+		mySSH.command('docker-compose --file ci-docker-compose.yml config --services | sed -e "s@^@service=@"', '\$', 2)
+		result = re.search('service=(?P<svc_name>[a-zA-Z0-9\_]+)', mySSH.getBefore())
+		if result is not None:
+			svcName = result.group('svc_name')
+			mySSH.command('docker-compose --file ci-docker-compose.yml up -d ' + svcName, '\$', 2)
+
+		# Checking Status
+		mySSH.command('docker-compose --file ci-docker-compose.yml config', '\$', 5)
+		result = re.search('container_name: (?P<container_name>[a-zA-Z0-9\-\_]+)', mySSH.getBefore())
+		unhealthyNb = 0
+		healthyNb = 0
+		startingNb = 0
+		containerName = ''
+		if result is not None:
+			containerName = result.group('container_name')
+			time.sleep(5)
+			cnt = 0
+			while (cnt < 3):
+				mySSH.command('docker inspect --format=\'{{.State.Health.Status}}\' ' + containerName, '\$', 5)
+				unhealthyNb = mySSH.getBefore().count('unhealthy')
+				healthyNb = mySSH.getBefore().count('healthy') - unhealthyNb
+				startingNb = mySSH.getBefore().count('starting')
+				if healthyNb == 1:
+					cnt = 10
+				else:
+					time.sleep(10)
+					cnt += 1
+		logging.debug(' -- ' + str(healthyNb) + ' healthy container(s)')
+		logging.debug(' -- ' + str(unhealthyNb) + ' unhealthy container(s)')
+		logging.debug(' -- ' + str(startingNb) + ' still starting container(s)')
+
+		status = False
+		if healthyNb == 1:
+			cnt = 0
+			while (cnt < 20):
+				mySSH.command('docker logs ' + containerName + ' | egrep --text --color=never -i "wait|sync|Starting"', '\$', 30) 
+				result = re.search('got sync|Starting F1AP at CU', mySSH.getBefore())
+				if result is None:
+					time.sleep(6)
+					cnt += 1
+				else:
+					cnt = 100
+					status = True
+					logging.info('\u001B[1m Deploying OAI object Pass\u001B[0m')
+					time.sleep(10)
+		mySSH.close()
+
+		if self.htmlObj is not None:
+			self.testCase_id = self.htmlObj.testCase_id
+		else:
+			self.testCase_id = '000000'
+		self.eNB_logFile[self.eNB_instance] = 'enb_' + self.testCase_id + '.log'
+
+		if self.htmlObj is not None:
+			if status:
+				self.htmlObj.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK)		
+			else:
+				self.htmlObj.CreateHtmlTestRow('N/A', 'KO', CONST.ALL_PROCESSES_OK)		
+
+	def UndeployObject(self):
+		logging.info('\u001B[1m Undeploying OAI Object Pass\u001B[0m')
+		if self.eNB_serverId[self.eNB_instance] == '0':
+			lIpAddr = self.eNBIPAddress
+			lUserName = self.eNBUserName
+			lPassWord = self.eNBPassword
+			lSourcePath = self.eNBSourceCodePath
+		elif self.eNB_serverId[self.eNB_instance] == '1':
+			lIpAddr = self.eNB1IPAddress
+			lUserName = self.eNB1UserName
+			lPassWord = self.eNB1Password
+			lSourcePath = self.eNB1SourceCodePath
+		elif self.eNB_serverId[self.eNB_instance] == '2':
+			lIpAddr = self.eNB2IPAddress
+			lUserName = self.eNB2UserName
+			lPassWord = self.eNB2Password
+			lSourcePath = self.eNB2SourceCodePath
+		if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '':
+			HELP.GenericHelp(CONST.Version)
+			sys.exit('Insufficient Parameter')
+		logging.debug('\u001B[1m Deploying OAI Object on server: ' + lIpAddr + '\u001B[0m')
+		mySSH = SSH.SSHConnection()
+		mySSH.open(lIpAddr, lUserName, lPassWord)
+		mySSH.command('cd ' + lSourcePath + '/' + self.yamlPath[self.eNB_instance], '\$', 5)
+		# Currently support only one
+		mySSH.command('docker-compose --file ci-docker-compose.yml config', '\$', 5)
+		result = re.search('container_name: (?P<container_name>[a-zA-Z0-9\-\_]+)', mySSH.getBefore())
+		if result is not None:
+			containerName = result.group('container_name')
+			mySSH.command('docker kill --signal INT ' + containerName, '\$', 30)
+			time.sleep(5)
+			mySSH.command('docker logs ' + containerName + ' > ' + lSourcePath + '/cmake_targets/' + self.eNB_logFile[self.eNB_instance], '\$', 30)
+			mySSH.command('docker rm -f ' + containerName, '\$', 30)
+
+		# Putting the CPUs back in a idle state, we do that only on a few servers
+		mySSH.command('hostname', '\$', 5)
+		result = re.search('obelix|asterix',  mySSH.getBefore())
+		if result is not None:
+			mySSH.command('if command -v cpupower &> /dev/null; then echo ' + lPassWord + ' | sudo -S cpupower idle-set -E; fi', '\$', 5)
+		mySSH.close()
+
+		# Analyzing log file!
+		if self.ranObj is not None:
+			copyin_res = mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/' + self.eNB_logFile[self.eNB_instance], '.')
+			nodeB_prefix = 'e'
+			if (copyin_res == -1):
+				if self.htmlObj is not None:
+					self.htmlObj.htmleNBFailureMsg='Could not copy ' + nodeB_prefix + 'NB logfile to analyze it!'
+					self.htmlObj.CreateHtmlTestRow('N/A', 'KO', CONST.ENB_PROCESS_NOLOGFILE_TO_ANALYZE)
+			else:
+				logging.debug('\u001B[1m Analyzing ' + nodeB_prefix + 'NB logfile \u001B[0m ' + self.eNB_logFile[self.eNB_instance])
+				logStatus = self.ranObj.AnalyzeLogFile_eNB(self.eNB_logFile[self.eNB_instance])
+				if (logStatus < 0):
+					if self.htmlObj is not None:
+						self.htmlObj.CreateHtmlTestRow(self.ranObj.runtime_stats, 'KO', logStatus)
+				else:
+					if self.htmlObj is not None:
+						self.htmlObj.CreateHtmlTestRow(self.ranObj.runtime_stats, 'OK', CONST.ALL_PROCESSES_OK)
+

ci-scripts/cls_oaicitest.py

@@ -216,7 +216,7 @@ class OaiCiTest():
 			HTML.CreateHtmlTabFooter(False)
 			self.ConditionalExit()
 
-	def CheckFlexranCtrlInstallation(self,RAN,EPC):
+	def CheckFlexranCtrlInstallation(self,RAN,EPC,CONTAINERS):
 		if EPC.IPAddress == '' or EPC.UserName == '' or EPC.Password == '':
 			return
 		SSH = sshconnection.SSHConnection()
@@ -234,6 +234,8 @@ class OaiCiTest():
 			if result is not None:
 				RAN.flexranCtrlDeployed=True
 				RAN.flexranCtrlIpAddress=result.group('flex_ip_addr')
+				CONTAINERS.flexranCtrlDeployed=True
+				CONTAINERS.flexranCtrlIpAddress=result.group('flex_ip_addr')
 				logging.debug('Flexran Controller is deployed: ' + RAN.flexranCtrlIpAddress)
 		SSH.close()
 

ci-scripts/main.py

@@ -310,6 +310,22 @@ def GetParametersFromXML(action):
 		if (string_field is not None):
 			EPC.yamlPath = string_field
 
+	elif action == 'Deploy_Object' or action == 'Undeploy_Object':
+		eNB_instance=test.findtext('eNB_instance')
+		if (eNB_instance is None):
+			CONTAINERS.eNB_instance=0
+		else:
+			CONTAINERS.eNB_instance=int(eNB_instance)
+		eNB_serverId=test.findtext('eNB_serverId')
+		if (eNB_serverId is None):
+			CONTAINERS.eNB_serverId[CONTAINERS.eNB_instance]='0'
+		else:
+			CONTAINERS.eNB_serverId[CONTAINERS.eNB_instance]=eNB_serverId
+		string_field = test.findtext('yaml_path')
+		if (string_field is not None):
+			CONTAINERS.yamlPath[CONTAINERS.eNB_instance] = string_field
+
+
 	else: # ie action == 'Run_PhySim':
 		ldpc.runargs = test.findtext('physim_run_args')
 		
@@ -368,6 +384,7 @@ RAN.htmlObj=HTML
 RAN.epcObj=EPC
 CONTAINERS.htmlObj=HTML
 CONTAINERS.epcObj=EPC
+CONTAINERS.ranObj=RAN
 
 
 ldpc=cls_physim.PhySim()    #create an instance for LDPC test using GPU or CPU build
@@ -591,7 +608,7 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re
 			logging.debug('ERROR: requested test is invalidly formatted: ' + test)
 			sys.exit(1)
 	if (EPC.IPAddress != '') and (EPC.IPAddress != 'none'):
-		CiTestObj.CheckFlexranCtrlInstallation(RAN,EPC)
+		CiTestObj.CheckFlexranCtrlInstallation(RAN,EPC,CONTAINERS)
 		EPC.SetMmeIPAddress()
 
 	#get the list of tests to be done
@@ -652,8 +669,6 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re
 							break
 				if action == 'Build_eNB':
 					RAN.BuildeNB()
-				elif action == 'Build_Image':
-					CONTAINERS.BuildImage()
 				elif action == 'WaitEndBuild_eNB':
 					RAN.WaitBuildeNBisFinished()
 				elif action == 'Initialize_eNB':
@@ -729,6 +744,12 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re
 					if ldpc.exitStatus==1:sys.exit()
 				elif action == 'Run_PhySim':
 					HTML=ldpc.Run_PhySim(HTML,CONST,id)
+				elif action == 'Build_Image':
+					CONTAINERS.BuildImage()
+				elif action == 'Deploy_Object':
+					CONTAINERS.DeployObject()
+				elif action == 'Undeploy_Object':
+					CONTAINERS.UndeployObject()
 				else:
 					sys.exit('Invalid class (action) from xml')
 				if not RAN.prematureExit:

ci-scripts/xml_class_list.yml

   - Build_PhySim
   - Run_PhySim
   - Build_eNB
-  - Build_Image
   - WaitEndBuild_eNB
   - Initialize_eNB
   - Terminate_eNB
@@ -35,3 +34,6 @@
   - Ping_CatM_module
   - IdleSleep
   - Perform_X2_Handover
+  - Build_Image
+  - Deploy_Object
+  - Undeploy_Object

ci-scripts/xml_files/fr1_usrp210_band7_deploy_test_05mhz_tm1.xml

+<!--
+
+ Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The OpenAirInterface Software Alliance licenses this file to You under
+ the OAI Public License, Version 1.1  (the "License"); you may not use this file
+ except in compliance with the License.
+ You may obtain a copy of the License at
+
+      http://www.openairinterface.org/?page_id=698
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+ For more information about the OpenAirInterface (OAI) Software Alliance:
+      contact@openairinterface.org
+
+-->
+<testCaseList>
+	<htmlTabRef>test-deploy-enb-mono</htmlTabRef>
+	<htmlTabName>Test-Deploy-eNB-Mono</htmlTabName>
+	<htmlTabIcon>tasks</htmlTabIcon>
+	<repeatCount>1</repeatCount>
+	<TestCaseRequestedList>
+ 040101
+ 030101 000020 040301 000021 040501 040601 040611 040641 040651 040401 040201 030201
+	</TestCaseRequestedList>
+	<TestCaseExclusionList></TestCaseExclusionList>
+
+	<testCase id="000001">
+		<class>IdleSleep</class>
+		<desc>Waiting for 60 seconds</desc>
+		<idle_sleep_time_in_sec>60</idle_sleep_time_in_sec>
+	</testCase>
+
+	<testCase id="000002">
+		<class>IdleSleep</class>
+		<desc>Waiting for 10 seconds</desc>
+		<idle_sleep_time_in_sec>10</idle_sleep_time_in_sec>
+	</testCase>
+
+    <testCase id="000020">
+        <class>CheckStatusUE</class>
+        <desc>Check UE(s) status before attachment</desc>
+        <expectedNbOfConnectedUEs>0</expectedNbOfConnectedUEs>
+    </testCase>
+
+    <testCase id="000021">
+        <class>CheckStatusUE</class>
+        <desc>Check UE(s) status after attachment</desc>
+        <expectedNbOfConnectedUEs>1</expectedNbOfConnectedUEs>
+    </testCase>
+
+	<testCase id="030101">
+		<class>Deploy_Object</class>
+		<desc>Deploy eNB (FDD/Band7/5MHz) in a container</desc>
+		<yaml_path>ci-scripts/yaml_files/fr1_enb_mono_fdd_tim</yaml_path>
+		<eNB_instance>0</eNB_instance>
+		<eNB_serverId>0</eNB_serverId>
+	</testCase>
+
+	<testCase id="030201">
+		<class>Undeploy_Object</class>
+		<desc>Undeploy eNB</desc>
+		<yaml_path>ci-scripts/yaml_files/fr1_enb_mono_fdd_tim</yaml_path>
+		<eNB_instance>0</eNB_instance>
+		<eNB_serverId>0</eNB_serverId>
+	</testCase>
+
+	<testCase id="040101">
+		<class>Initialize_UE</class>
+		<desc>Initialize UE</desc>
+	</testCase>
+
+	<testCase id="040201">
+		<class>Terminate_UE</class>
+		<desc>Terminate UE</desc>
+	</testCase>
+
+	<testCase id="040301">
+		<class>Attach_UE</class>
+		<desc>Attach UE</desc>
+	</testCase>
+
+	<testCase id="040401">
+		<class>Detach_UE</class>
+		<desc>Detach UE</desc>
+	</testCase>
+
+	<testCase id="040501">
+		<class>Ping</class>
+		<desc>ping (5MHz - 20 sec)</desc>
+		<ping_args>-c 20</ping_args>
+		<ping_packetloss_threshold>5</ping_packetloss_threshold>
+	</testCase>
+
+	<testCase id="040601">
+		<class>Iperf</class>
+		<desc>iperf (5MHz - DL/15Mbps/UDP)(30 sec)</desc>
+		<iperf_args>-u -b 15M -t 30 -i 1</iperf_args>
+		<iperf_packetloss_threshold>50</iperf_packetloss_threshold>
+		<iperf_profile>single-ue</iperf_profile>
+	</testCase>
+
+	<testCase id="040611">
+		<class>Iperf</class>
+		<desc>iperf (5MHz - DL/TCP)(30 sec)</desc>
+		<iperf_args>-t 30 -i 1</iperf_args>
+		<iperf_packetloss_threshold>50</iperf_packetloss_threshold>
+		<iperf_profile>single-ue</iperf_profile>
+	</testCase>
+
+	<testCase id="040641">
+		<class>Iperf</class>
+		<desc>iperf (5MHz - UL/7.5Mbps/UDP)(30 sec)</desc>
+		<iperf_args>-u -b 7.5M -t 30 -i 1 -R</iperf_args>
+		<iperf_packetloss_threshold>50</iperf_packetloss_threshold>
+		<iperf_profile>single-ue</iperf_profile>
+	</testCase>
+
+	<testCase id="040651">
+		<class>Iperf</class>
+		<desc>iperf (5MHz - UL/TCP)(30 sec)</desc>
+		<iperf_args>-t 30 -i 1 -R</iperf_args>
+		<iperf_packetloss_threshold>50</iperf_packetloss_threshold>
+		<iperf_profile>single-ue</iperf_profile>
+	</testCase>
+
+</testCaseList>

ci-scripts/yaml_files/fr1_enb_mono_fdd_tim/docker-compose.yml

@@ -33,6 +33,12 @@ services:
         networks:
             public_net:
                 ipv4_address: 192.168.61.30
+        healthcheck:
+            # pgrep does NOT work
+            test: /bin/bash -c "ps aux | grep -v grep | grep -c softmodem"
+            interval: 10s
+            timeout: 5s
+            retries: 5
 
 networks:
     public_net: