feat(ci): reporting in HTML of formatting and licences checks

Signed-off-by: Raphael Defosseux <raphael.defosseux@eurecom.fr>

ci-scripts/cls_static_code_analysis.py

@@ -37,6 +37,7 @@ import logging
 import os
 from pathlib import Path
 import time
+from multiprocessing import Process, Lock, SimpleQueue
 
 #-----------------------------------------------------------
 # OAI Testing modules
@@ -275,15 +276,167 @@ class StaticCodeAnalysis():
 			mySSH.command('git checkout -f ' + self.ranCommitID, '\$', 30)
 		# if the branch is not develop, then it is a merge request and we need to do
 		# the potential merge. Note that merge conflicts should already been checked earlier
+		argToPass = ''
 		if (self.ranAllowMerge):
+			argToPass = '--build-arg MERGE_REQUEST=true --build-arg SRC_BRANCH=' + self.ranBranch
 			if self.ranTargetBranch == '':
 				if (self.ranBranch != 'develop') and (self.ranBranch != 'origin/develop'):
 					mySSH.command('git merge --ff origin/develop -m "Temporary merge for CI"', '\$', 5)
+					argToPass += ' --build-arg TARGET_BRANCH=develop '
 			else:
 				logging.debug('Merging with the target branch: ' + self.ranTargetBranch)
 				mySSH.command('git merge --ff origin/' + self.ranTargetBranch + ' -m "Temporary merge for CI"', '\$', 5)
+				argToPass += ' --build-arg TARGET_BRANCH=' + self.ranTargetBranch + ' '
 
 		mySSH.command('docker image rm oai-formatting-check:latest || true', '\$', 60)
-		mySSH.command('docker build --target oai-formatting-check --tag oai-formatting-check:latest --file ci-scripts/docker/Dockerfile.formatting.bionic . > cmake_targets/log/oai-formatting-check.txt 2>&1', '\$', 600)
+		mySSH.command('docker build --target oai-formatting-check --tag oai-formatting-check:latest ' + argToPass + '--file ci-scripts/docker/Dockerfile.formatting.bionic . > cmake_targets/log/oai-formatting-check.txt 2>&1', '\$', 600)
 
-		return 0
+		mySSH.command('docker image rm oai-formatting-check:latest || true', '\$', 60)
+		mySSH.command('docker image prune --force', '\$', 60)
+		mySSH.command('docker volume prune --force', '\$', 60)
+
+		# Analyzing the logs
+		mySSH.command('cd ' + lSourcePath + '/cmake_targets', '\$', 5)
+		mySSH.command('mkdir -p build_log_' + self.testCase_id, '\$', 5)
+		mySSH.command('mv log/* ' + 'build_log_' + self.testCase_id, '\$', 5)
+		mySSH.close()
+
+		mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/build_log_' + self.testCase_id + '/*', '.')
+
+		finalStatus = 0
+		if (os.path.isfile('./oai-formatting-check.txt')):
+			analyzed = False
+			nbFilesNotFormatted = 0
+			listFiles = False
+			listFilesNotFormatted = []
+			circularHeaderDependency = False
+			circularHeaderDependencyFiles = []
+			gnuGplLicence = False
+			gnuGplLicenceFiles = []
+			suspectLicence = False
+			suspectLicenceFiles = []
+			with open('./oai-formatting-check.txt', 'r') as logfile:
+				for line in logfile:
+					ret = re.search('./ci-scripts/checkCodingFormattingRules.sh', str(line))
+					if ret is not None:
+						analyzed = True
+					if analyzed:
+						ret = re.search('Nb Files that do NOT follow OAI rules: (?P<nb_errors>[0-9\.]+)', str(line))
+						if ret is not None:
+							nbFilesNotFormatted = int(ret.group('nb_errors'))
+
+						if re.search('=== Files not properly formatted ===', str(line)) is not None:
+							listFiles = True
+						if listFiles:
+							if re.search('Removing intermediate container', str(line)) is not None:
+								listFiles = False
+							elif re.search('Running in|Files not properly formatted', str(line)) is not None:
+								pass
+							else:
+								listFilesNotFormatted.append(str(line).strip())
+
+						if re.search('=== Files with incorrect define protection ===', str(line)) is not None:
+							circularHeaderDependency = True
+						if circularHeaderDependency:
+							if re.search('Removing intermediate container', str(line)) is not None:
+								circularHeaderDependency = False
+							elif re.search('Running in|Files with incorrect define protection', str(line)) is not None:
+								pass
+							else:
+								circularHeaderDependencyFiles.append(str(line).strip())
+
+						if re.search('=== Files with a GNU GPL licence Banner ===', str(line)) is not None:
+							gnuGplLicence = True
+						if gnuGplLicence:
+							if re.search('Removing intermediate container', str(line)) is not None:
+								gnuGplLicence = False
+							elif re.search('Running in|Files with a GNU GPL licence Banner', str(line)) is not None:
+								pass
+							else:
+								gnuGplLicenceFiles.append(str(line).strip())
+
+						if re.search('=== Files with a suspect Banner ===', str(line)) is not None:
+							suspectLicence = True
+						if suspectLicence:
+							if re.search('Removing intermediate container', str(line)) is not None:
+								suspectLicence = False
+							elif re.search('Running in|Files with a suspect Banner', str(line)) is not None:
+								pass
+							else:
+								suspectLicenceFiles.append(str(line).strip())
+
+				logfile.close()
+			if analyzed:
+				logging.debug('files not formatted properly: ' + str(nbFilesNotFormatted))
+				if nbFilesNotFormatted == 0:
+					HTML.CreateHtmlTestRow('File(s) Format', 'OK', CONST.ALL_PROCESSES_OK)
+				else:
+					html_queue = SimpleQueue()
+					html_cell = '<pre style="background-color:white">\n'
+					html_cell += 'Number of files not following OAI Rules: ' + str(nbFilesNotFormatted) + '\n'
+					for nFile in listFilesNotFormatted:
+						html_cell += str(nFile).strip() + '\n'
+					html_cell += '</pre>'
+					html_queue.put(html_cell)
+					HTML.CreateHtmlTestRowQueue('File(s) Format', 'KO', 1, html_queue)
+					del(html_cell)
+					del(html_queue)
+
+				logging.debug('header files not respecting the circular dependency protection: ' + str(len(circularHeaderDependencyFiles)))
+				if len(circularHeaderDependencyFiles) == 0:
+					HTML.CreateHtmlTestRow('Header Circular Dependency', 'OK', CONST.ALL_PROCESSES_OK)
+				else:
+					html_queue = SimpleQueue()
+					html_cell = '<pre style="background-color:white">\n'
+					html_cell += 'Number of files not respecting: ' + str(len(circularHeaderDependencyFiles)) + '\n'
+					for nFile in circularHeaderDependencyFiles:
+						html_cell += str(nFile).strip() + '\n'
+					html_cell += '</pre>'
+					html_queue.put(html_cell)
+					HTML.CreateHtmlTestRowQueue('Header Circular Dependency', 'KO', 1, html_queue)
+					del(html_cell)
+					del(html_queue)
+					finalStatus = -1
+
+				logging.debug('files with a GNU GPL license: ' + str(len(gnuGplLicenceFiles)))
+				if len(gnuGplLicenceFiles) == 0:
+					HTML.CreateHtmlTestRow('Files w/ GNU GPL License', 'OK', CONST.ALL_PROCESSES_OK)
+				else:
+					html_queue = SimpleQueue()
+					html_cell = '<pre style="background-color:white">\n'
+					html_cell += 'Number of files not respecting: ' + str(len(gnuGplLicenceFiles)) + '\n'
+					for nFile in gnuGplLicenceFiles:
+						html_cell += str(nFile).strip() + '\n'
+					html_cell += '</pre>'
+					html_queue.put(html_cell)
+					HTML.CreateHtmlTestRowQueue('Files w/ GNU GPL License', 'KO', 1, html_queue)
+					del(html_cell)
+					del(html_queue)
+					finalStatus = -1
+
+				logging.debug('files with a suspect license: ' + str(len(suspectLicenceFiles)))
+				if len(suspectLicenceFiles) == 0:
+					HTML.CreateHtmlTestRow('Files with suspect license', 'OK', CONST.ALL_PROCESSES_OK)
+				else:
+					html_queue = SimpleQueue()
+					html_cell = '<pre style="background-color:white">\n'
+					html_cell += 'Number of files not respecting: ' + str(len(suspectLicenceFiles)) + '\n'
+					for nFile in suspectLicenceFiles:
+						html_cell += str(nFile).strip() + '\n'
+					html_cell += '</pre>'
+					html_queue.put(html_cell)
+					HTML.CreateHtmlTestRowQueue('Files with suspect license', 'KO', 1, html_queue)
+					del(html_cell)
+					del(html_queue)
+					finalStatus = -1
+
+			else:
+				finalStatus = -1
+				HTML.htmleNBFailureMsg = 'Could not fully analyze oai-formatting-check.txt file'
+				HTML.CreateHtmlTestRow('N/A', 'KO', CONST.ENB_PROCESS_NOLOGFILE_TO_ANALYZE)
+		else:
+			finalStatus = -1
+			HTML.htmleNBFailureMsg = 'Could not access oai-formatting-check.txt file'
+			HTML.CreateHtmlTestRow('N/A', 'KO', CONST.ENB_PROCESS_NOLOGFILE_TO_ANALYZE)
+
+		return finalStatus

ci-scripts/docker/Dockerfile.formatting.bionic

@@ -16,10 +16,16 @@ WORKDIR /oai-ran
 COPY . .
 
 RUN /bin/bash -c "if [[ -v MERGE_REQUEST ]]; then echo 'Source Branch = $SRC_BRANCH'; echo 'Target Branch = $TARGET_BRANCH'; else echo 'Push to develop'; fi"
-RUN /bin/bash -c "if [[ -v MERGE_REQUEST ]]; then ./ci-scripts/checkCodingFormattingRules.sh --src-branch $SRC_BRANCH --target-branch $TARGET_BRANCH; else ./ci-scripts/checkCodingFormattingRules.sh; fi" && \
-   ls -lst *.txt && \
-   cat *.txt
+RUN /bin/bash -c "if [[ -v MERGE_REQUEST ]]; then ./ci-scripts/checkCodingFormattingRules.sh --src-branch $SRC_BRANCH --target-branch $TARGET_BRANCH; else ./ci-scripts/checkCodingFormattingRules.sh; fi"
 
-#RUN ./ci-scripts/checkCodingFormattingRules.sh && \
-#    ls -lst *.txt
-#
+RUN echo "=== Files not properly formatted ===" && \
+    /bin/bash -c "if [[ -f oai_rules_result_list.txt ]]; then cat oai_rules_result_list.txt; fi"
+
+RUN echo "=== Files with incorrect define protection ===" && \
+   /bin/bash -c "if [[ -f header-files-w-incorrect-define.txt ]]; then cat header-files-w-incorrect-define.txt; fi"
+
+RUN echo "=== Files with a GNU GPL licence Banner ===" && \
+   /bin/bash -c "if [[ -f files-w-gnu-gpl-license-banner.txt ]]; then cat files-w-gnu-gpl-license-banner.txt; fi"
+
+RUN echo "=== Files with a suspect Banner ===" && \
+  /bin/bash -c "if [[ -f files-w-suspect-banner.txt ]]; then cat files-w-suspect-banner.txt; fi"

ci-scripts/main.py

@@ -924,7 +924,9 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re
 				elif action == 'Cppcheck_Analysis':
 					SCA.CppCheckAnalysis(HTML)
 				elif action == 'LicenceAndFormattingCheck':
-					SCA.LicenceAndFormattingCheck(HTML)
+					ret = SCA.LicenceAndFormattingCheck(HTML)
+					if ret != 0:
+						RAN.prematureExit = True
 				elif action == 'Deploy_Run_PhySim':
 					PHYSIM.Deploy_PhySim(HTML, RAN)
 				elif action == 'DeployGenObject':