[CI][Docker] Building docker images in python framework

- only on Ubuntu and Docker
- No back ground computing to paralellize RHEL/Ubuntu
- No image clean up
Signed-off-by: Raphael Defosseux <raphael.defosseux@eurecom.fr>

ci-scripts/main.py

@@ -98,8 +98,9 @@ def AssignParams(params_dict):
 
 
 def GetParametersFromXML(action):
-	if action == 'Build_eNB':
+	if action == 'Build_eNB' or action == 'Build_Image':
 		RAN.Build_eNB_args=test.findtext('Build_eNB_args')
+		RAN.imageKind=test.findtext('kind')
 		forced_workspace_cleanup = test.findtext('forced_workspace_cleanup')
 		if (forced_workspace_cleanup is None):
 			RAN.Build_eNB_forced_workspace_cleanup=False
@@ -127,8 +128,9 @@ def GetParametersFromXML(action):
 			else:
 				RAN.backgroundBuild=False
 
-	elif action == 'WaitEndBuild_eNB':
+	elif action == 'WaitEndBuild_eNB' or action == 'WaitEndBuild_Image':
 		RAN.Build_eNB_args=test.findtext('Build_eNB_args')
+		RAN.imageKind=test.findtext('kind')
 		eNB_instance=test.findtext('eNB_instance')
 		if (eNB_instance is None):
 			RAN.eNB_instance=0
@@ -640,8 +642,12 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re
 							break
 				if action == 'Build_eNB':
 					RAN.BuildeNB()
+				elif action == 'Build_Image':
+					RAN.BuildImage()
 				elif action == 'WaitEndBuild_eNB':
 					RAN.WaitBuildeNBisFinished()
+				elif action == 'WaitEndBuild_Image':
+					RAN.WaitBuildImageisFinished()
 				elif action == 'Initialize_eNB':
 					check_eNB = False
 					check_OAI_UE = False

ci-scripts/ran.py

@@ -77,6 +77,7 @@ class RANManagement():
 		self.backgroundBuildTestId = ['', '', '']
 		self.Build_eNB_forced_workspace_cleanup = False
 		self.Initialize_eNB_args = ''
+		self.imageKind = ''
 		self.air_interface = ['', '', ''] #changed from 'lte' to '' may lead to side effects in main
 		self.eNB_instance = 0
 		self.eNB_serverId = ['', '', '']
@@ -318,6 +319,171 @@ class RANManagement():
 				self.htmlObj.CreateHtmlTabFooter(False)
 			sys.exit(1)
 
+	def BuildImage(self):
+		if self.ranRepository == '' or self.ranBranch == '' or self.ranCommitID == '':
+			HELP.GenericHelp(CONST.Version)
+			sys.exit('Insufficient Parameter')
+		if self.eNB_serverId[self.eNB_instance] == '0':
+			lIpAddr = self.eNBIPAddress
+			lUserName = self.eNBUserName
+			lPassWord = self.eNBPassword
+			lSourcePath = self.eNBSourceCodePath
+		elif self.eNB_serverId[self.eNB_instance] == '1':
+			lIpAddr = self.eNB1IPAddress
+			lUserName = self.eNB1UserName
+			lPassWord = self.eNB1Password
+			lSourcePath = self.eNB1SourceCodePath
+		elif self.eNB_serverId[self.eNB_instance] == '2':
+			lIpAddr = self.eNB2IPAddress
+			lUserName = self.eNB2UserName
+			lPassWord = self.eNB2Password
+			lSourcePath = self.eNB2SourceCodePath
+		if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '':
+			HELP.GenericHelp(CONST.Version)
+			sys.exit('Insufficient Parameter')
+		logging.debug('Building on server: ' + lIpAddr)
+		mySSH = SSH.SSHConnection()
+		mySSH.open(lIpAddr, lUserName, lPassWord)
+
+		imageNames = []
+		result = re.search('eNB', self.imageKind)
+		if result is not None:
+			imageNames.append(('oai-enb', 'eNB'))
+		else:
+			result = re.search('gNB', self.imageKind)
+			if result is not None:
+				imageNames.append(('oai-gnb', 'gNB'))
+			else:
+				result = re.search('all', self.imageKind)
+				if result is not None:
+					imageNames.append(('oai-enb', 'eNB'))
+					imageNames.append(('oai-gnb', 'gNB'))
+		if len(imageNames) == 0:
+			imageNames.append(('oai-enb', 'eNB'))
+		# Workaround for some servers, we need to erase completely the workspace
+		if self.Build_eNB_forced_workspace_cleanup:
+			mySSH.command('echo ' + lPassWord + ' | sudo -S rm -Rf ' + lSourcePath, '\$', 15)
+		if self.htmlObj is not None:
+			self.testCase_id = self.htmlObj.testCase_id
+		else:
+			self.testCase_id = '000000'
+		# on RedHat/CentOS .git extension is mandatory
+		result = re.search('([a-zA-Z0-9\:\-\.\/])+\.git', self.ranRepository)
+		if result is not None:
+			full_ran_repo_name = self.ranRepository
+		else:
+			full_ran_repo_name = self.ranRepository + '.git'
+		mySSH.command('mkdir -p ' + lSourcePath, '\$', 5)
+		mySSH.command('cd ' + lSourcePath, '\$', 5)
+		mySSH.command('if [ ! -e .git ]; then stdbuf -o0 git clone ' + full_ran_repo_name + ' .; else stdbuf -o0 git fetch --prune; fi', '\$', 600)
+		# Raphael: here add a check if git clone or git fetch went smoothly
+		mySSH.command('git config user.email "jenkins@openairinterface.org"', '\$', 5)
+		mySSH.command('git config user.name "OAI Jenkins"', '\$', 5)
+
+		mySSH.command('echo ' + lPassWord + ' | sudo -S git clean -x -d -ff', '\$', 30)
+		mySSH.command('mkdir -p cmake_targets/log', '\$', 5)
+		# if the commit ID is provided use it to point to it
+		if self.ranCommitID != '':
+			mySSH.command('git checkout -f ' + self.ranCommitID, '\$', 5)
+		# if the branch is not develop, then it is a merge request and we need to do 
+		# the potential merge. Note that merge conflicts should already been checked earlier
+		imageTag = 'develop'
+		if (self.ranAllowMerge):
+			imageTag = 'ci-temp'
+			if self.ranTargetBranch == '':
+				if (self.ranBranch != 'develop') and (self.ranBranch != 'origin/develop'):
+					mySSH.command('git merge --ff origin/develop -m "Temporary merge for CI"', '\$', 5)
+			else:
+				logging.debug('Merging with the target branch: ' + self.ranTargetBranch)
+				mySSH.command('git merge --ff origin/' + self.ranTargetBranch + ' -m "Temporary merge for CI"', '\$', 5)
+		# Let's remove any previous run artifacts if still there
+		mySSH.command('docker image prune --force', '\$', 5)
+		mySSH.command('docker image rm ran-build:' + imageTag, '\$', 5)
+		for image,pattern in imageNames:
+			mySSH.command('docker image rm ' + image + ':' + imageTag, '\$', 5)
+		# Build the shared image
+		mySSH.command('docker build --target ran-build --tag ran-build:' + imageTag + ' --file docker/Dockerfile.ran.ubuntu18 --build-arg NEEDED_GIT_PROXY="http://proxy.eurecom.fr:8080" . > cmake_targets/log/ran-build.log 2>&1', '\$', 800)
+		# Build the target image(s)
+		previousImage='ran-build:' + imageTag
+		danglingShaOnes=[]
+		for image,pattern in imageNames:
+			mySSH.command('docker build --target ' + image + ' --tag ' + image + ':' + imageTag + ' --file docker/Dockerfile.' + pattern + '.ubuntu18 . > cmake_targets/log/' + image + '.log 2>&1', '\$', 600)
+			# Retrieving the dangling image(s) for the log collection
+			mySSH.command('docker images --filter "dangling=true" --filter "since=' + previousImage + '" -q | sed -e "s#^#sha=#"', '\$', 5)
+			result = re.search('sha=(?P<imageShaOne>[a-zA-Z0-9\-\_]+)', mySSH.getBefore())
+			if result is not None:
+				danglingShaOnes.append((image, result.group('imageShaOne')))
+			previousImage=image + ':' + imageTag
+
+		imageTag = 'ci-temp'
+		# First verify if images were properly created.
+		status = True
+		mySSH.command('docker image inspect --format=\'Size = {{.Size}} bytes\' ran-build:' + imageTag, '\$', 5)
+		if mySSH.getBefore().count('No such object') != 0:
+			logging.error('Could not build properly ran-build')
+			status = False
+		else:
+			result = re.search('Size = (?P<size>[0-9\-]+) bytes', mySSH.getBefore())
+			if result is not None:
+				imageSize = float(result.group('size'))
+				imageSize = imageSize / 1000
+				if imageSize < 1000:
+					logging.debug('\u001B[1m   ran-build size is ' + ('%.0f' % imageSize) + ' kbytes\u001B[0m')
+				else:
+					imageSize = imageSize / 1000
+					if imageSize < 1000:
+						logging.debug('\u001B[1m   ran-build size is ' + ('%.0f' % imageSize) + ' Mbytes\u001B[0m')
+					else:
+						imageSize = imageSize / 1000
+						logging.debug('\u001B[1m   ran-build size is ' + ('%.3f' % imageSize) + ' Gbytes\u001B[0m')
+			else:
+				logging.debug('ran-build size is unknown')
+		for image,pattern in imageNames:
+			mySSH.command('docker image inspect --format=\'Size = {{.Size}} bytes\' ' + image + ':' + imageTag, '\$', 5)
+			if mySSH.getBefore().count('No such object') != 0:
+				logging.error('Could not build properly ' + image)
+				status = False
+			else:
+				result = re.search('Size = (?P<size>[0-9\-]+) bytes', mySSH.getBefore())
+				if result is not None:
+					imageSize = float(result.group('size'))
+					imageSize = imageSize / 1000
+					if imageSize < 1000:
+						logging.debug('\u001B[1m   ' + image + ' size is ' + ('%.0f' % imageSize) + ' kbytes\u001B[0m')
+					else:
+						imageSize = imageSize / 1000
+						if imageSize < 1000:
+							logging.debug('\u001B[1m   ' + image + ' size is ' + ('%.0f' % imageSize) + ' Mbytes\u001B[0m')
+						else:
+							imageSize = imageSize / 1000
+							logging.debug('\u001B[1m   ' + image + ' size is ' + ('%.3f' % imageSize) + ' Gbytes\u001B[0m')
+				else:
+					logging.debug('ran-build size is unknown')
+		if not status:
+			mySSH.close()
+			logging.error('\u001B[1m Building OAI Images Failed\u001B[0m')
+			if self.htmlObj is not None:
+				self.htmlObj.CreateHtmlTestRow(self.imageKind, 'KO', CONST.ALL_PROCESSES_OK)
+				self.htmlObj.CreateHtmlTabFooter(False)
+			sys.exit(1)
+
+		# Recover build logs, for the moment only possible when build is successful
+		mySSH.command('docker create --name test ran-build:' + imageTag, '\$', 5)
+		mySSH.command('mkdir -p cmake_targets/log/ran-build', '\$', 5)
+		mySSH.command('docker cp test:/oai-ran/cmake_targets/log/. /tmp/CI-eNB/cmake_targets/log/ran-build', '\$', 5)
+		mySSH.command('docker rm -f test', '\$', 5)
+		for image,shaone in danglingShaOnes:
+			mySSH.command('mkdir -p cmake_targets/log/' + image, '\$', 5)
+			mySSH.command('docker create --name test ' + shaone, '\$', 5)
+			mySSH.command('docker cp test:/oai-ran/cmake_targets/log/. /tmp/CI-eNB/cmake_targets/log/' + image, '\$', 5)
+			mySSH.command('docker rm -f test', '\$', 5)
+		mySSH.command('docker image prune --force', '\$', 5)
+		mySSH.close()
+
+		logging.info('\u001B[1m Building OAI Image(s) Pass\u001B[0m')
+		if self.htmlObj is not None:
+			self.htmlObj.CreateHtmlTestRow(self.imageKind, 'OK', CONST.ALL_PROCESSES_OK)		
+		
 	def InitializeeNB(self):
 		if self.eNB_serverId[self.eNB_instance] == '0':
 			lIpAddr = self.eNBIPAddress

ci-scripts/xml_class_list.yml

   - Build_PhySim
   - Run_PhySim
   - Build_eNB
+  - Build_Image
   - WaitEndBuild_eNB
+  - WaitEndBuild_Image
   - Initialize_eNB
   - Terminate_eNB
   - Initialize_UE

ci-scripts/xml_files/fr1_image_build.xml

+<!--
+
+ Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The OpenAirInterface Software Alliance licenses this file to You under
+ the OAI Public License, Version 1.1  (the "License"); you may not use this file
+ except in compliance with the License.
+ You may obtain a copy of the License at
+
+      http://www.openairinterface.org/?page_id=698
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+ For more information about the OpenAirInterface (OAI) Software Alliance:
+      contact@openairinterface.org
+
+ 000004
+-->
+<testCaseList>
+	<htmlTabRef>build-tab</htmlTabRef>
+	<htmlTabName>Build</htmlTabName>
+	<htmlTabIcon>wrench</htmlTabIcon>
+	<TestCaseRequestedList>
+ 000001
+	</TestCaseRequestedList>
+	<TestCaseExclusionList></TestCaseExclusionList>
+
+	<testCase id="000001">
+		<class>Build_Image</class>
+		<desc>Build eNB Image</desc>
+		<kind>all</kind>
+		<eNB_instance>0</eNB_instance>
+		<eNB_serverId>0</eNB_serverId>
+		<backgroundBuild>True</backgroundBuild>
+	</testCase>
+
+	<testCase id="000004">
+		<class>WaitEndBuild_Image</class>
+		<desc>Wait for end of Build eNB</desc>
+		<kind>eNB</kind>
+		<eNB_instance>0</eNB_instance>
+		<eNB_serverId>0</eNB_serverId>
+	</testCase>
+
+</testCaseList>