[CI][DOCKER] handling the case of build not successful

Signed-off-by: Raphael Defosseux <raphael.defosseux@eurecom.fr>

ci-scripts/cls_containerize.py

@@ -192,10 +192,11 @@ class Containerize():
 
 		sharedimage = 'ran-build'
 		# Let's remove any previous run artifacts if still there
-		mySSH.command(self.cli + ' image prune --force', '\$', 5)
-		mySSH.command(self.cli + ' image rm ' + sharedimage + ':' + imageTag, '\$', 5)
+		mySSH.command(self.cli + ' image prune --force', '\$', 30)
+		mySSH.command(self.cli + ' image rm ' + sharedimage + ':' + imageTag, '\$', 30)
 		for image,pattern in imageNames:
-			mySSH.command(self.cli + ' image rm ' + image + ':' + imageTag, '\$', 5)
+			mySSH.command(self.cli + ' image rm ' + image + ':' + imageTag, '\$', 30)
+
 		# Build the shared image
 		mySSH.command(self.cli + ' build --target ' + sharedimage + ' --tag ' + sharedimage + ':' + imageTag + ' --file docker/Dockerfile.ran' + self.dockerfileprefix + ' --build-arg NEEDED_GIT_PROXY="http://proxy.eurecom.fr:8080" . > cmake_targets/log/ran-build.log 2>&1', '\$', 1600)
 		# First verify if the shared image was properly created.
@@ -225,31 +226,35 @@ class Containerize():
 				logging.debug('ran-build size is unknown')
 		# If the shared image failed, no need to continue
 		if not status:
+			# Recover the name of the failed container?
+			mySSH.command(self.cli + ' image prune --force', '\$', 30)
 			mySSH.close()
 			logging.error('\u001B[1m Building OAI Images Failed\u001B[0m')
 			HTML.CreateHtmlTestRow(self.imageKind, 'KO', CONST.ALL_PROCESSES_OK)
 			HTML.CreateHtmlTabFooter(False)
 			sys.exit(1)
+		else:
+			# Recover build logs, for the moment only possible when build is successful
+			mySSH.command(self.cli + ' create --name test ' + sharedimage + ':' + imageTag, '\$', 5)
+			mySSH.command('mkdir -p cmake_targets/log/ran-build', '\$', 5)
+			mySSH.command(self.cli + ' cp test:/oai-ran/cmake_targets/log/. cmake_targets/log/ran-build', '\$', 5)
+			mySSH.command(self.cli + ' rm -f test', '\$', 5)
 
 		# Build the target image(s)
-		previousImage = sharedimage + ':' + imageTag
-		danglingShaOnes=[]
 		for image,pattern in imageNames:
 			# the archived Dockerfiles have "ran-build:latest" as base image
 			# we need to update them with proper tag
 			mySSH.command('sed -i -e "s#' + sharedimage + ':latest#' + sharedimage + ':' + imageTag + '#" docker/Dockerfile.' + pattern + self.dockerfileprefix, '\$', 5)
 			mySSH.command(self.cli + ' build --target ' + image + ' --tag ' + image + ':' + imageTag + ' --file docker/Dockerfile.' + pattern + self.dockerfileprefix + ' . > cmake_targets/log/' + image + '.log 2>&1', '\$', 1200)
-			# Retrieving the dangling image(s) for the log collection
-			mySSH.command(self.cli + ' images --filter "dangling=true" --filter "since=' + previousImage + '" -q | sed -e "s#^#sha=#"', '\$', 5)
-			result = re.search('sha=(?P<imageShaOne>[a-zA-Z0-9\-\_]+)', mySSH.getBefore())
-			if result is not None:
-				danglingShaOnes.append((image, result.group('imageShaOne')))
-			previousImage = image + ':' + imageTag
 			# checking the status of the build
 			mySSH.command(self.cli + ' image inspect --format=\'Size = {{.Size}} bytes\' ' + image + ':' + imageTag, '\$', 5)
+			# split the log
+			mySSH.command('mkdir -p cmake_targets/log/' + image, '\$', 5)
+			mySSH.command('python3 ci-scripts/docker_log_split.py --logfilename=cmake_targets/log/' + image + '.log', '\$', 5)
 			if (mySSH.getBefore().count('No such object') != 0) or (mySSH.getBefore().count('no such image') != 0):
 				logging.error('Could not build properly ' + image)
 				status = False
+				# Here we should check if the last container corresponds to a failed command and destroy it
 			else:
 				result = re.search('Size *= *(?P<size>[0-9\-]+) *bytes', mySSH.getBefore())
 				if result is not None:
@@ -269,27 +274,11 @@ class Containerize():
 							self.allImagesSize[image] = str(round(imageSize,1)) + ' Gbytes'
 				else:
 					logging.debug('ran-build size is unknown')
-		if not status:
-			mySSH.close()
-			logging.error('\u001B[1m Building OAI Images Failed\u001B[0m')
-			HTML.CreateHtmlTestRow(self.imageKind, 'KO', CONST.ALL_PROCESSES_OK)
-			#HTML.CreateHtmlNextTabHeaderTestRow(self.collectInfo, self.allImagesSize)
-			HTML.CreateHtmlTabFooter(False)
-			sys.exit(1)
+			# Now pruning dangling images in between target builds
+			mySSH.command(self.cli + ' image prune --force', '\$', 30)
 
-		# Recover build logs, for the moment only possible when build is successful
-		mySSH.command(self.cli + ' create --name test ' + sharedimage + ':' + imageTag, '\$', 5)
-		mySSH.command('mkdir -p cmake_targets/log/ran-build', '\$', 5)
-		mySSH.command(self.cli + ' cp test:/oai-ran/cmake_targets/log/. cmake_targets/log/ran-build', '\$', 5)
-		mySSH.command(self.cli + ' rm -f test', '\$', 5)
-		for image,shaone in danglingShaOnes:
-			mySSH.command('mkdir -p cmake_targets/log/' + image, '\$', 5)
-			mySSH.command(self.cli + ' create --name test ' + shaone, '\$', 5)
-			mySSH.command(self.cli + ' cp test:/oai-ran/cmake_targets/log/. cmake_targets/log/' + image, '\$', 5)
-			mySSH.command(self.cli + ' rm -f test', '\$', 5)
-		mySSH.command(self.cli + ' image prune --force', '\$', 5)
-		mySSH.command('cd cmake_targets', '\$', 5)
-		mySSH.command('mkdir -p build_log_' + self.testCase_id, '\$', 5)
+		# Analyzing the logs
+		mySSH.command('cd ' + lSourcePath + '/cmake_targets', '\$', 5)
 		mySSH.command('mv log/* ' + 'build_log_' + self.testCase_id, '\$', 5)
 
 		mySSH.command('cd ' + lSourcePath + '/cmake_targets', '\$', 5)
@@ -335,9 +324,16 @@ class Containerize():
 				files[fil] = errorandwarnings
 			self.collectInfo[image] = files
 		
+		if status:
 			logging.info('\u001B[1m Building OAI Image(s) Pass\u001B[0m')
 			HTML.CreateHtmlTestRow(self.imageKind, 'OK', CONST.ALL_PROCESSES_OK)
 			HTML.CreateHtmlNextTabHeaderTestRow(self.collectInfo, self.allImagesSize)
+		else:
+			logging.error('\u001B[1m Building OAI Images Failed\u001B[0m')
+			HTML.CreateHtmlTestRow(self.imageKind, 'KO', CONST.ALL_PROCESSES_OK)
+			HTML.CreateHtmlNextTabHeaderTestRow(self.collectInfo, self.allImagesSize)
+            HTML.CreateHtmlTabFooter(False)
+            sys.exit(1)
 
 	def DeployObject(self, HTML, EPC):
 		if self.eNB_serverId[self.eNB_instance] == '0':

ci-scripts/docker_log_split.py

+#/*
+# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
+# * contributor license agreements.  See the NOTICE file distributed with
+# * this work for additional information regarding copyright ownership.
+# * The OpenAirInterface Software Alliance licenses this file to You under
+# * the OAI Public License, Version 1.1  (the "License"); you may not use this file
+# * except in compliance with the License.
+# * You may obtain a copy of the License at
+# *
+# *      http://www.openairinterface.org/?page_id=698
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# *-------------------------------------------------------------------------------
+# * For more information about the OpenAirInterface (OAI) Software Alliance:
+# *      contact@openairinterface.org
+# */
+#---------------------------------------------------------------------
+# Python for CI of OAI-eNB + COTS-UE
+#
+#   Required Python Version
+#     Python 3.x
+#
+#   Required Python Package
+#     pexpect
+#---------------------------------------------------------------------
+
+
+#-----------------------------------------------------------
+# Import Libs
+#-----------------------------------------------------------
+import sys		# arg
+import re		# reg
+import os
+import subprocess
+
+
+class SplitReport():
+	def __init__(self):
+		self.logfilename = ''
+		self.destinationFolder = ''
+
+	def split(self):
+		self.destinationFolder = self.logfilename.replace(".log","")
+		if os.path.isfile(self.logfilename):
+			newImageLog = open(self.logfilename + '.new', 'w')
+			copyFlag = True
+			with open(self.logfilename, 'r') as imageLog:
+				for line in imageLog:
+					header = False
+					ret = re.search('====== Start of log for ([0-9\.A-Za-z\-\_]+) ======', line)
+					if ret is not None:
+						copyFlag = False
+						header = True
+						detailedLogFile = open(self.destinationFolder + '/' + ret.group(1), 'w')
+					if copyFlag:
+						newImageLog.write(line)
+					ret = re.search('====== End of log for ([0-9\.A-Za-z\-\_]+) ======', line)
+					if ret is not None:
+						copyFlag = True
+						detailedLogFile.close()
+					elif not copyFlag and not header:
+						detailedLogFile.write(line)
+			imageLog.close()
+			newImageLog.close()
+			os.rename(self.logfilename + '.new', self.logfilename)
+		else:
+			print('Cannot split unfound file')
+
+#--------------------------------------------------------------------------------------------------------
+#
+# Start of main
+#
+#--------------------------------------------------------------------------------------------------------
+
+argvs = sys.argv
+argc = len(argvs)
+
+SP = SplitReport()
+
+while len(argvs) > 1:
+	myArgv = argvs.pop(1)
+	if re.match('^\-\-logfilename=(.+)$', myArgv, re.IGNORECASE):
+		matchReg = re.match('^\-\-logfilename=(.+)$', myArgv, re.IGNORECASE)
+		SP.logfilename = matchReg.group(1)
+
+SP.split()
+
+sys.exit(0)