Reimplement Native Build (CI step Build_eNB)

The BuildeNB steps either does things that are already implemented
(created workspace), or things that we don't use (separate build_wait),
and is generally quite convoluted when all it does is calling build_oai.
Reimplement by reusing Create_Workspace, and make build_oai in a single
step. To clean up, make a new module in which this function does not
need a class instantiation. Instead, we give all necessary parameters
directly.

ci-scripts/cls_native.py

+#/*
+# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
+# * contributor license agreements.  See the NOTICE file distributed with
+# * this work for additional information regarding copyright ownership.
+# * The OpenAirInterface Software Alliance licenses this file to You under
+# * the OAI Public License, Version 1.1  (the "License"); you may not use this file
+# * except in compliance with the License.
+# * You may obtain a copy of the License at
+# *
+# *      http://www.openairinterface.org/?page_id=698
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# *-------------------------------------------------------------------------------
+# * For more information about the OpenAirInterface (OAI) Software Alliance:
+# *      contact@openairinterface.org
+# */
+#---------------------------------------------------------------------
+
+import logging
+import re
+
+import cls_cmd
+import cls_oai_html
+import constants as CONST
+
+class Native():
+
+	def Build(test_case, HTML, host, directory, options):
+		logging.debug(f'Building on server: {host}')
+		HTML.testCase_id = test_case
+
+		with cls_cmd.getConnection(host) as ssh:
+			base = f"{directory}/cmake_targets"
+			ret = ssh.run(f"{base}/build_oai {options} > {base}/log/build_oai.log", timeout=900)
+			success = ret.returncode == 0
+			logs = ssh.run(f"cat {base}/log/build_oai.log", silent=True)
+			logging.debug(f"build finished with code {ret.returncode}, output:\n{logs.stdout}")
+
+			# create log directory, and copy build logs
+			target = f"{base}/build_log_{test_case}/"
+			ssh.run(f"mkdir -p {target}")
+			ssh.run(f"mv {base}/log/* {target}")
+
+			# check if build artifacts are there
+			# NOTE: build_oai should fail with exit code if it could not build, but it does not
+			build_dir = f"{base}/ran_build/build"
+			build_gnb = re.search('--gNB', options) is not None
+			if build_gnb:
+				success = success and ssh.run(f"ls {build_dir}/nr-softmodem").returncode == 0
+			build_enb = re.search('--eNB', options) is not None
+			if build_enb:
+				success = success and ssh.run(f"ls {build_dir}/lte-softmodem").returncode == 0
+
+		if success:
+			logging.info('\u001B[1m Building OAI Pass\u001B[0m')
+			HTML.CreateHtmlTestRow(options, 'OK', CONST.ALL_PROCESSES_OK)
+		else:
+			logging.error('\u001B[1m Building OAI Failed\u001B[0m')
+			HTML.CreateHtmlTestRow(options, 'KO', CONST.ALL_PROCESSES_OK)
+		return success

ci-scripts/main.py

@@ -44,6 +44,7 @@ import cls_containerize	 #class Containerize for all container-based operations
 import cls_static_code_analysis  #class for static code analysis
 import cls_physim1		 #class PhySim for physical simulators deploy and run
 import cls_cluster		 # class for building/deploying on cluster
+import cls_native        # class for all native/source-based operations
 
 import sshconnection 
 import epc
@@ -149,7 +150,7 @@ def ExecuteActionWithParam(action):
 		if proxy_commit is not None:
 			CONTAINERS.proxyCommit = proxy_commit
 		if action == 'Build_eNB':
-			success = RAN.BuildeNB(HTML)
+			success = cls_native.Native.Build(HTML.testCase_id, HTML, RAN.eNBIPAddress, RAN.eNBSourceCodePath, RAN.Build_eNB_args)
 		elif action == 'Build_Image':
 			success = CONTAINERS.BuildImage(HTML)
 		elif action == 'Build_Proxy':

ci-scripts/ran.py

@@ -105,185 +105,6 @@ class RANManagement():
 # RAN management functions
 #-----------------------------------------------------------
 
-	def BuildeNB(self, HTML):
-		if self.ranRepository == '' or self.ranBranch == '' or self.ranCommitID == '':
-			HELP.GenericHelp(CONST.Version)
-			sys.exit('Insufficient Parameter')
-		if self.eNB_serverId[self.eNB_instance] == '0':
-			lIpAddr = self.eNBIPAddress
-			lUserName = self.eNBUserName
-			lPassWord = self.eNBPassword
-			lSourcePath = self.eNBSourceCodePath
-		elif self.eNB_serverId[self.eNB_instance] == '1':
-			lIpAddr = self.eNB1IPAddress
-			lUserName = self.eNB1UserName
-			lPassWord = self.eNB1Password
-			lSourcePath = self.eNB1SourceCodePath
-		elif self.eNB_serverId[self.eNB_instance] == '2':
-			lIpAddr = self.eNB2IPAddress
-			lUserName = self.eNB2UserName
-			lPassWord = self.eNB2Password
-			lSourcePath = self.eNB2SourceCodePath
-		if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '':
-			HELP.GenericHelp(CONST.Version)
-			sys.exit('Insufficient Parameter')
-		logging.debug('Building on server: ' + lIpAddr)
-		mySSH = SSH.SSHConnection()
-		mySSH.open(lIpAddr, lUserName, lPassWord)
-		
-		# Check if we build an 5G-NR gNB or an LTE eNB
-		result = re.search('--RU', self.Build_eNB_args)
-		if result is not None:
-			self.air_interface[self.eNB_instance] = 'oairu'
-		else:
-			result = re.search('--gNB', self.Build_eNB_args)
-			if result is not None:
-				self.air_interface[self.eNB_instance] = 'nr-softmodem'
-			else:
-				self.air_interface[self.eNB_instance] = 'lte-softmodem'
-		
-		# Worakround for some servers, we need to erase completely the workspace
-		if self.Build_eNB_forced_workspace_cleanup:
-			mySSH.command('echo ' + lPassWord + ' | sudo -S rm -Rf ' + lSourcePath, '\$', 15)
-		self.testCase_id = HTML.testCase_id
-		# on RedHat/CentOS .git extension is mandatory
-		result = re.search('([a-zA-Z0-9\:\-\.\/])+\.git', self.ranRepository)
-		if result is not None:
-			full_ran_repo_name = self.ranRepository.replace('git/', 'git')
-		else:
-			full_ran_repo_name = self.ranRepository + '.git'
-		mySSH.command('mkdir -p ' + lSourcePath, '\$', 5)
-		mySSH.command('cd ' + lSourcePath, '\$', 5)
-		mySSH.command('if [ ! -e .git ]; then stdbuf -o0 git clone ' + full_ran_repo_name + ' .; else stdbuf -o0 git fetch --prune; fi', '\$', 600)
-		# Raphael: here add a check if git clone or git fetch went smoothly
-		mySSH.command('git config user.email "jenkins@openairinterface.org"', '\$', 5)
-		mySSH.command('git config user.name "OAI Jenkins"', '\$', 5)
-		mySSH.command('git advice.detachedHead false', '\$', 5)
-		# Checking the BUILD INFO file
-		if not self.backgroundBuild:
-			mySSH.command('ls *.txt', '\$', 5)
-			result = re.search('LAST_BUILD_INFO', mySSH.getBefore())
-			if result is not None:
-				mismatch = False
-				mySSH.command('grep --colour=never SRC_COMMIT LAST_BUILD_INFO.txt', '\$', 2)
-				result = re.search(self.ranCommitID, mySSH.getBefore())
-				if result is None:
-					mismatch = True
-				mySSH.command('grep --colour=never MERGED_W_TGT_BRANCH LAST_BUILD_INFO.txt', '\$', 2)
-				if (self.ranAllowMerge):
-					result = re.search('YES', mySSH.getBefore())
-					if result is None:
-						mismatch = True
-					mySSH.command('grep --colour=never TGT_BRANCH LAST_BUILD_INFO.txt', '\$', 2)
-					if self.ranTargetBranch == '':
-						result = re.search('develop', mySSH.getBefore())
-					else:
-						result = re.search(self.ranTargetBranch, mySSH.getBefore())
-					if result is None:
-						mismatch = True
-				else:
-					result = re.search('NO', mySSH.getBefore())
-					if result is None:
-						mismatch = True
-				if not mismatch:
-					mySSH.close()
-					HTML.CreateHtmlTestRow(self.Build_eNB_args, 'OK', CONST.ALL_PROCESSES_OK)
-					return True
-
-		mySSH.command('echo ' + lPassWord + ' | sudo -S git clean -x -d -ff', '\$', 30)
-		# if the commit ID is provided use it to point to it
-		if self.ranCommitID != '':
-			mySSH.command('git checkout -f ' + self.ranCommitID, '\$', 30)
-		# if the branch is not develop, then it is a merge request and we need to do 
-		# the potential merge. Note that merge conflicts should already been checked earlier
-		if (self.ranAllowMerge):
-			if self.ranTargetBranch == '':
-				if (self.ranBranch != 'develop') and (self.ranBranch != 'origin/develop'):
-					mySSH.command('git merge --ff origin/develop -m "Temporary merge for CI"', '\$', 30)
-			else:
-				logging.debug('Merging with the target branch: ' + self.ranTargetBranch)
-				mySSH.command('git merge --ff origin/' + self.ranTargetBranch + ' -m "Temporary merge for CI"', '\$', 30)
-		logging.debug(mySSH.getBefore()) # print what git said when merging/checking out
-		mySSH.command('source oaienv', '\$', 5)
-		mySSH.command('cd cmake_targets', '\$', 5)
-		mySSH.command('mkdir -p log', '\$', 5)
-		mySSH.command('chmod 777 log', '\$', 5)
-		# no need to remove in log (git clean did the trick)
-		if self.backgroundBuild:
-			mySSH.command('echo "./build_oai ' + self.Build_eNB_args + '" > ./my-lte-softmodem-build.sh', '\$', 5)
-			mySSH.command('chmod 775 ./my-lte-softmodem-build.sh', '\$', 5)
-			mySSH.command('echo ' + lPassWord + ' | sudo -S ls', '\$', 5)
-			logging.debug(mySSH.getBefore()) # print current directory contents for verification
-			mySSH.command('echo $USER; nohup sudo -E ./my-lte-softmodem-build.sh' + ' > ' + lSourcePath + '/cmake_targets/compile_oai_enb.log ' + ' 2>&1 &', lUserName, 5)
-			mySSH.close()
-			HTML.CreateHtmlTestRow(self.Build_eNB_args, 'OK', CONST.ALL_PROCESSES_OK)
-			self.backgroundBuildTestId[int(self.eNB_instance)] = self.testCase_id
-			return True
-		mySSH.command('stdbuf -o0 ./build_oai ' + self.Build_eNB_args + ' 2>&1 | stdbuf -o0 tee compile_oai_enb.log', 'BUILD SHOULD BE SUCCESSFUL|build have failed', 900)
-		mySSH.close()
-		return self.checkBuildeNB(lIpAddr, lUserName, lPassWord, lSourcePath, self.testCase_id, HTML)
-
-	def checkBuildeNB(self, lIpAddr, lUserName, lPassWord, lSourcePath, testcaseId, HTML):
-		HTML.testCase_id=testcaseId
-
-		mySSH = SSH.SSHConnection()
-		mySSH.open(lIpAddr, lUserName, lPassWord)
-		mySSH.command('cd ' + lSourcePath + '/cmake_targets', '\$', 3)
-		mySSH.command('ls ran_build/build', '\$', 3)
-		mySSH.command('ls ran_build/build', '\$', 3)
-
-		#check if we have the build corresponding to the air interface keywords (nr-softmode, lte-softmodem)
-		logging.info('CHECK Build with IP='+lIpAddr+' SourcePath='+lSourcePath)
-		result = re.search(self.air_interface[self.eNB_instance], mySSH.getBefore())
-		if result is None:
-			buildStatus = False #if not, build failed
-		else:
-			buildStatus = True 
-			# Generating a BUILD INFO file
-			mySSH.command('echo "SRC_BRANCH: ' + self.ranBranch + '" > ../LAST_BUILD_INFO.txt', '\$', 2)
-			mySSH.command('echo "SRC_COMMIT: ' + self.ranCommitID + '" >> ../LAST_BUILD_INFO.txt', '\$', 2)
-			if (self.ranAllowMerge):
-				mySSH.command('echo "MERGED_W_TGT_BRANCH: YES" >> ../LAST_BUILD_INFO.txt', '\$', 2)
-				if self.ranTargetBranch == '':
-					mySSH.command('echo "TGT_BRANCH: develop" >> ../LAST_BUILD_INFO.txt', '\$', 2)
-				else:
-					mySSH.command('echo "TGT_BRANCH: ' + self.ranTargetBranch + '" >> ../LAST_BUILD_INFO.txt', '\$', 2)
-			else:
-				mySSH.command('echo "MERGED_W_TGT_BRANCH: NO" >> ../LAST_BUILD_INFO.txt', '\$', 2)
-				
-				
-		mySSH.command('mkdir -p build_log_' + testcaseId, '\$', 5)
-		mySSH.command('mv log/* ' + 'build_log_' + testcaseId, '\$', 5)
-		mySSH.command('mv compile_oai_enb.log ' + 'build_log_' + testcaseId, '\$', 5)
-		if self.eNB_serverId[self.eNB_instance] != '0':
-			mySSH.command('cd cmake_targets', '\$', 5)
-			mySSH.command('if [ -e tmp_build' + testcaseId + '.zip ]; then rm -f tmp_build' + testcaseId + '.zip; fi', '\$', 5)
-			mySSH.command('zip -r -qq tmp_build' + testcaseId + '.zip build_log_' + testcaseId, '\$', 5)
-			mySSH.close()
-			if (os.path.isfile('./tmp_build' + testcaseId + '.zip')):
-				os.remove('./tmp_build' + testcaseId + '.zip')
-			mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/tmp_build' + testcaseId + '.zip', '.')
-			if (os.path.isfile('./tmp_build' + testcaseId + '.zip')):
-				mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, './tmp_build' + testcaseId + '.zip', self.eNBSourceCodePath + '/cmake_targets/.')
-				os.remove('./tmp_build' + testcaseId + '.zip')
-				mySSH.open(self.eNBIPAddress, self.eNBUserName, self.eNBPassword)
-				mySSH.command('cd ' + self.eNBSourceCodePath + '/cmake_targets', '\$', 5)
-				#-qq quiet / -u update orcreate files
-				mySSH.command('unzip -o -u -qq -DD tmp_build' + testcaseId + '.zip', '\$', 5)
-				mySSH.command('rm -f tmp_build' + testcaseId + '.zip', '\$', 5)
-				mySSH.close()
-		else:
-			mySSH.close()
-
-		#generate logging info depending on buildStatus and air interface
-		if buildStatus:
-			logging.info('\u001B[1m Building OAI ' + self.air_interface[self.eNB_instance] + ' Pass\u001B[0m')
-			HTML.CreateHtmlTestRow(self.Build_eNB_args, 'OK', CONST.ALL_PROCESSES_OK)
-		else:
-			logging.error('\u001B[1m Building OAI ' + self.air_interface[self.eNB_instance] + ' Failed\u001B[0m')
-			HTML.CreateHtmlTestRow(self.Build_eNB_args, 'KO', CONST.ALL_PROCESSES_OK)
-		return buildStatus
-
 	def InitializeeNB(self, HTML, EPC):
 		if self.eNB_serverId[self.eNB_instance] == '0':
 			lIpAddr = self.eNBIPAddress

ci-scripts/xml_files/gnb_usrp_build.xml

@@ -26,15 +26,21 @@
 	<htmlTabIcon>wrench</htmlTabIcon>
 	<TestCaseRequestedList>
 100000
+000000
 010101
 	</TestCaseRequestedList>
 	<TestCaseExclusionList></TestCaseExclusionList>
 
+	<testCase id="000000">
+		<class>Create_Workspace</class>
+		<desc>Creating new Workspace</desc>
+	</testCase>
+
 	<testCase id="010101">
 		<mode>TesteNB</mode>	
 		<class>Build_eNB</class>
 		<desc>Build gNB (USRP)</desc>
-		<Build_eNB_args>--gNB -w USRP --ninja -c -P --build-lib "ldpc_cuda ldpc_t2" --cmake-opt -DASN1C_EXEC=/opt/asn1c/bin/asn1c</Build_eNB_args>
+		<Build_eNB_args>--gNB -w USRP --ninja -c -P --build-lib ldpc_cuda --build-lib ldpc_t2 --cmake-opt -DASN1C_EXEC=/opt/asn1c/bin/asn1c</Build_eNB_args>
 		<forced_workspace_cleanup>True</forced_workspace_cleanup>
 	</testCase>