CI: bracket testing added

Signed-off-by: ismail <mohammed.ismail@openairinterface.org>

ci-scripts/Jenkinsfile-GitLab-Docker

@@ -338,6 +338,35 @@ pipeline {
         }
       }
     }
+    stage ('Bracket Testing with DsTester') {
+      steps {
+        script {
+          gitlabCommitStatus(name: "Bracket Test with DsTester") {
+            localStatus = build job: params.NRF_BT_PipelineName,
+              parameters: [
+                string(name: 'NRF_TAG', value: String.valueOf(nrf_tag))
+              ], propagate: false
+            localResult = localStatus.getResult()
+
+            if (localStatus.resultIsBetterOrEqualTo('SUCCESS')) {
+              echo "Bracket Test Job is OK"
+            } else {
+              echo "Bracket Test Job is KO"
+              sh "ci-scripts/fail.sh"
+            }
+          }
+        }
+      }
+      post {
+        always {
+          script {
+            copyArtifacts(projectName: params.AMF_BT_PipelineName,
+                          filter: '*_results_oai_nrf_bt.html',
+                          selector: lastCompleted())
+          }
+        }
+      }
+    }
     stage ('Testing in CN-5G-FED environment') {
       steps {
         script {
@@ -439,6 +468,10 @@ pipeline {
             sh "sed -i -e 's#TEMPLATE_MERGE_REQUEST_LINK#${gitlabMergeRequestLink}#g' *_results_oai_cn5g.html"
             sh "sed -i -e 's#TEMPLATE_MERGE_REQUEST_TEMPLATE#${env.gitlabMergeRequestTitle}#' *_results_oai_cn5g.html"
           }
+          if (fileExists('test_results_oai_nrf_bt.html')) {
+            sh "sed -i -e 's#TEMPLATE_MERGE_REQUEST_LINK#${gitlabMergeRequestLink}#g' *_results_oai_nrf_bt.html"
+            sh "sed -i -e 's#TEMPLATE_MERGE_REQUEST_TEMPLATE#${env.gitlabMergeRequestTitle}#' *_results_oai_nrf_bt.html"
+          }
         } else {
           sh "python3 ci-scripts/generateHtmlReport.py --job_name=${JOB_NAME} --job_id=${BUILD_ID} --job_url=${BUILD_URL} --git_url=${GIT_URL} --git_src_branch=${GIT_BRANCH} --git_src_commit=${GIT_COMMIT}"
         }
@@ -451,6 +484,10 @@ pipeline {
           sh "sed -i -e 's#TEMPLATE_TIME#${JOB_TIMESTAMP}#' *_results_oai_cn5g.html"
           archiveArtifacts artifacts: '*_results_oai_cn5g.html'
         }
+        if (fileExists('test_results_oai_nrf_bt.html')) {
+          sh "sed -i -e 's#TEMPLATE_TIME#${JOB_TIMESTAMP}#' *_results_oai_nrf_bt.html"
+          archiveArtifacts artifacts: '*_results_oai_nrf_bt.html'
+        }
 
         // Sending email to commiter
         if (params.sendToCommitterEmail != null) {

ci-scripts/dsTesterDockerCompose/docker-compose.tplt

+version: '3.8'
+services:
+    cicd_oai_nrf:
+        container_name: cicd-oai-nrf
+        image: oai-nrf:NRF_IMAGE_TAG
+        ports:
+            - 80
+            - 9090
+        command: >
+            bash -c "nohup tshark -i eth0 -w /tmp/nrf.pcap 2>&1 > /dev/null &
+            /openair-nrf/bin/oai_nrf -c /openair-nrf/etc/nrf.conf -o | tee /tmp/nrf.log 2>&1
+            "
+        cap_add:
+            - NET_ADMIN
+        environment:
+            - TZ=Europe/Paris
+            - NRF_INTERFACE_NAME_FOR_SBI=eth0
+            - NRF_INTERFACE_PORT_FOR_SBI=80
+            - NRF_INTERFACE_HTTP2_PORT_FOR_SBI=9090
+            - NRF_API_VERSION=v1
+            - INSTANCE=0
+            - PID_DIRECTORY=/var/run
+        networks:
+            cicd_public_net:
+                ipv4_address: 192.168.61.195
+        volumes:
+            - ./nrf-healthy-check.sh:/openair-nrf/bin/nrf-healthy-check.sh
+        healthcheck:
+            test: /bin/bash -c "/openair-nrf/bin/nrf-healthy-check.sh"
+            interval: 10s
+            timeout: 5s
+            retries: 5
+
+networks:
+    cicd_public_net:
+        name: cicd-oai-public-net
+        driver: bridge
+        ipam:
+            config:
+                - subnet: 192.168.61.192/26

ci-scripts/dsTesterDockerCompose/nrf-healthy-check.sh

+#!/bin/bash
+
+STATUS=0
+
+RESULT=$(ps aux | grep -v nohup || true)
+SUB='/openair-nrf/bin/oai_nrf -c /openair-nrf/etc/nrf.conf -o'
+if [[ $RESULT =~ $SUB ]]; then
+    STATUS=0
+else
+   STATUS=-1
+fi
+
+exit $STATUS

ci-scripts/routeCheck.py

+#/*
+# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
+# * contributor license agreements.  See the NOTICE file distributed with
+# * this work for additional information regarding copyright ownership.
+# * The OpenAirInterface Software Alliance licenses this file to You under
+# * the OAI Public License, Version 1.1  (the "License"); you may not use this file
+# * except in compliance with the License.
+# * You may obtain a copy of the License at
+# *
+# *   http://www.openairinterface.org/?page_id=698
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# *-------------------------------------------------------------------------------
+# * For more information about the OpenAirInterface (OAI) Software Alliance:
+# *   contact@openairinterface.org
+# */
+#---------------------------------------------------------------------
+
+import os
+import re
+import sys
+import subprocess
+import yaml
+
+class IpRouteCheck():
+	def __init__(self):
+		self.mode = ''
+		self.userName = ''
+		self.hostName = ''
+		self.subnet = ''
+		self.gatwayIp = ''
+		self.interfaceName = ''
+
+	def getSubnet(self):
+		cmd = "egrep 'subnet' ci-scripts/dsTesterDockerCompose/docker-compose.yml"
+		ret = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding='utf-8')
+		if ret.returncode == 0:
+			if ret.stdout is not None:
+				result = re.search("(?:[0-9]{1,3}[.]){3}[0-9]{1,3}/[0-9]{1,2}", ret.stdout.strip())
+				if result is not None:
+					self.subnet = result.group(0)
+					#print("found subnet:", self.subnet)
+			else:
+				print("subnet not found in docker compose")
+				sys.exit(-1)
+		else:
+			print("docker-compose file not found")
+			sys.exit(-1)
+
+	def getGatwayIp(self):
+		cmd = "ifconfig | grep 192.168.18"
+		ret = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding='utf-8')
+		if ret.returncode == 0:
+			result = re.search("inet (?:[0-9]{1,3}[.]){3}[0-9]{1,3}", ret.stdout.strip())
+			self.gatwayIp = result.group(0)
+			#print("Gatway IP Address:", self.gatwayIp)
+		else:
+			print("No Gatway IP address starts with 196.168.18")
+			sys.exit(-1)	
+
+	def routeCheck(self):
+		# Get the interface name
+		cmd = "ip route | grep 192.168.18.0/24 | awk {'print $3'}"
+		ret = subprocess.run(f'ssh {self.userName}@{self.hostName} {cmd} || true', shell=True, stdout=subprocess.PIPE, encoding='utf-8')
+		#print('interface name:', ret.stdout.strip())
+		self.interfaceName = ret.stdout.strip()
+		# Check whether or not ip route exist 
+		cmd = f'ip route | grep -c "{self.subnet}"'
+		ret = subprocess.run(f'ssh {self.userName}@{self.hostName} {cmd} || true', shell=True, stdout=subprocess.PIPE, encoding='utf-8')
+		if ret.stdout is not None:
+			if ret.stdout.strip() == '1':
+				#print('Route exits')
+				if IPRC.mode == 'Delete':
+					IPRC.routeDel()
+				else:
+					sys.exit(0)
+			else:
+				#print("Route not found")
+				if IPRC.mode == 'Add':
+					IPRC.routeAdd()
+				else:
+					sys.exit(0)
+
+	def routeAdd(self):
+		# Add the route
+		cmd = f"sudo ip route add {self.subnet} via {self.gatwayIp} dev {self.interfaceName}"
+		ret = subprocess.run(f'ssh {self.userName}@{self.hostName} {cmd} || true', shell=True, stdout=subprocess.PIPE, encoding='utf-8')
+		print("Added ip route")
+
+	def routeDel(self):
+		# Delete the route
+		cmd = f"sudo ip route del {self.subnet} via {self.gatwayIp} dev {self.interfaceName}"
+		ret = subprocess.run(f'ssh {self.userName}@{self.hostName} {cmd} || true', shell=True, stdout=subprocess.PIPE, encoding='utf-8')
+		print("Deleted ip route")
+
+def Usage():
+	print('----------------------------------------------------------------------------------------------------------------------')
+	print('routecheck.py')
+	print('   Add and Delete the ip route on the Server.')
+	print('----------------------------------------------------------------------------------------------------------------------')
+	print('Usage: python3 routecheck.py [options]')
+	print('  --help  Show this help.')
+	print('---------------------------------------------------------------------------------------------- Mandatory Options -----')
+	print('  --mode=[Add/Delete]')
+	print('  --userName=[server userName where to add/delete]')
+	print('  --hostName=[server hostName where to add/delete]')
+	print('------------------------------------------------------------------------------------------------- Actions Syntax -----')
+	print('python3 routeCheck.py --mode=Add [Mandatory Options]')
+	print('python3 routeCheck.py --mode=Delete [Mandatory Options]')	
+
+
+#--------------------------------------------------------------------------------------------------------
+#
+# Start of main
+#
+#--------------------------------------------------------------------------------------------------------
+
+argvs = sys.argv
+
+IPRC = IpRouteCheck()
+
+while len(argvs) > 1:
+	myArgv = argvs.pop(1)
+	if re.match('^\-\-help$', myArgv, re.IGNORECASE):
+		Usage()
+		sys.exit(0)
+	elif re.match('^\-\-mode=(.+)$', myArgv, re.IGNORECASE):
+		matchReg = re.match('^\-\-mode=(.+)$', myArgv, re.IGNORECASE)
+		IPRC.mode = matchReg.group(1)
+	elif re.match('^\-\-userName=(.+)$', myArgv, re.IGNORECASE):
+		matchReg = re.match('^\-\-userName=(.+)$', myArgv, re.IGNORECASE)
+		IPRC.userName = matchReg.group(1)
+	elif re.match('^\-\-hostName=(.+)$', myArgv, re.IGNORECASE):
+		matchReg = re.match('^\-\-hostName=(.+)$', myArgv, re.IGNORECASE)
+		IPRC.hostName = matchReg.group(1)
+	else:
+		sys.exit('Invalid Parameter: ' + myArgv)
+
+if IPRC.mode == '' or IPRC.userName == '' or IPRC.hostName == '':
+	sys.exit('Missing Parameter in job description')
+
+IPRC.getSubnet()
+IPRC.getGatwayIp()
+IPRC.routeCheck()

ci-scripts/toCheckDSTesterResult.py

+#/*
+# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
+# * contributor license agreements.  See the NOTICE file distributed with
+# * this work for additional information regarding copyright ownership.
+# * The OpenAirInterface Software Alliance licenses this file to You under
+# * the OAI Public License, Version 1.1  (the "License"); you may not use this file
+# * except in compliance with the License.
+# * You may obtain a copy of the License at
+# *
+# *   http://www.openairinterface.org/?page_id=698
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# *-------------------------------------------------------------------------------
+# * For more information about the OpenAirInterface (OAI) Software Alliance:
+# *   contact@openairinterface.org
+# */
+#---------------------------------------------------------------------
+
+import re
+import sys
+import subprocess
+import yaml
+import os
+import time
+
+locexist = False
+cwd = os.getcwd()
+try:
+    with open(cwd + '/DS-TEST-RESULTS/dsTester_Summary.txt') as f:
+        for line in f:
+            if re.search('Result file is available here', str(line)):
+                result = re.search('(?:\/.+?\/)(.+?)(?:\/.+)', str(line))
+                if result:
+                    result1 = re.search('^(.*/)([^/]*)$', str(result.group(0)))
+                    subprocess.check_output(f'cp {result1.group(1)}* DS-TEST-RESULTS/', stderr=subprocess.STDOUT, shell=True, universal_newlines=True)
+                    result2 = re.search('.*\/(.+)$', str(result.group(0)))
+                    filename = result2.group(1)
+                locexist = True
+except IOError:
+    sys.exit("File not accessible to check DSTester Summary: DS-TEST-RESULTS/dsTester_Summary.txt")
+
+if locexist:
+    try:
+        with open(cwd + f'/DS-TEST-RESULTS/{filename}') as f:
+            data = yaml.load(f)
+            if data["final-result"] == 'fail':
+                sys.exit('DsTester final result FAILED')
+    except IOError:
+        sys.exit(f'File not accessible to check DSTester result: DS-TEST-RESULTS/{filename}')