Merge branch 'ci-sanity-check-enhancements' into 'develop'

CI: Sanity Check Enhancements

See merge request oai/cn5g/oai-cn5g-upf-vpp!9

CHANGELOG.md

 # RELEASE NOTES: #
 
-## v1.2.0 -- September 2021 ##
+## v1.2.0 -- October 2021 ##
 
 * Initial Public Release
 * Full support for Ubuntu18 and RHEL7

ci-scripts/Jenkinsfile-GitLab-Docker

@@ -163,7 +163,7 @@ pipeline {
                 myShCmd('docker image prune --force', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
                 // Currently this pipeline only runs for pushes to `develop` branch
                 // First clean image registry
-                myShCmd('docker image rm oai-upf:' + upf_tag + ' || true', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
+                myShCmd('docker image rm oai-upf-vpp:' + upf_tag + ' || true', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
                 myShCmd('mkdir -p archives', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
                 myShCmd('docker build --no-cache --target oai-upf-vpp --tag oai-upf-vpp:' + upf_tag + ' --file docker/Dockerfile.upf-vpp.ubuntu18 --build-arg NEEDED_GIT_PROXY="http://proxy.eurecom.fr:8080" . > archives/upf_docker_image_build.log 2>&1', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
                 myShCmd('docker image ls >> archives/upf_docker_image_build.log', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
@@ -266,7 +266,7 @@ pipeline {
             ]) {
               myShCmd("echo ${DH_Password} | docker login --username ${DH_Username} --password-stdin", rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
               myShCmd("docker image tag oai-upf-vpp:develop ${DH_Username}/oai-upf-vpp:develop", rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
-              myShCmd("docker push ${DH_Username}/oai-upf-vpp:develop", rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
+              myShCmd("docker push --quiet ${DH_Username}/oai-upf-vpp:develop", rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
               myShCmd("docker rmi ${DH_Username}/oai-upf-vpp:develop", rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
               myShCmd("docker logout", rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
             }
@@ -299,14 +299,14 @@ pipeline {
         // Removing temporary / intermediate images
         try {
           if ("MERGE".equals(env.gitlabActionType)) {
-            myShCmd('docker image rm --force oai-upf:ci-tmp', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
+            myShCmd('docker image rm --force oai-upf-vpp:ci-tmp', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
           }
         } catch (Exception e) {
           echo "We failed to delete the OAI-UPF-VPP temp image"
         }
         try {
           if ("MERGE".equals(env.gitlabActionType)) {
-            myShCmd('sudo podman image rm oai-upf:ci-tmp', rem_rhel_host_flag, rem_rhel_host_user, rem_rhel_host)
+            myShCmd('sudo podman image rm oai-upf-vpp:ci-tmp', rem_rhel_host_flag, rem_rhel_host_user, rem_rhel_host)
           }
         } catch (Exception e) {
           echo "We failed to delete the OAI-UPF-VPP temp image"
@@ -334,7 +334,7 @@ pipeline {
         if ("MERGE".equals(env.gitlabActionType)) {
           sh "python3 ci-scripts/generateHtmlReport.py --job_name=${JOB_NAME} --job_id=${BUILD_ID} --job_url=${BUILD_URL} --git_url=${GIT_URL} --git_src_branch=${env.gitlabSourceBranch} --git_src_commit=${env.gitlabMergeRequestLastCommit} --git_pull_request=True --git_target_branch=${env.gitlabTargetBranch} --git_target_commit=${GIT_COMMIT}"
           if (fileExists('test_results_oai_upf.html')) {
-//            sh "sed -i -e 's#TEMPLATE_MERGE_REQUEST_LINK#${gitlabMergeRequestLink}#g' test_results_oai_upf.html"
+            sh "sed -i -e 's#TEMPLATE_MERGE_REQUEST_LINK#${gitlabMergeRequestLink}#g' test_results_oai_upf.html"
             sh "sed -i -e 's#TEMPLATE_MERGE_REQUEST_TEMPLATE#${env.gitlabMergeRequestTitle}#' test_results_oai_upf.html"
           }
         } else {

ci-scripts/generateHtmlReport.py

-#/*
-# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
-# * contributor license agreements.  See the NOTICE file distributed with
-# * this work for additional information regarding copyright ownership.
-# * The OpenAirInterface Software Alliance licenses this file to You under
-# * the OAI Public License, Version 1.1  (the "License"); you may not use this file
-# * except in compliance with the License.
-# * You may obtain a copy of the License at
-# *
-# *	  http://www.openairinterface.org/?page_id=698
-# *
-# * Unless required by applicable law or agreed to in writing, software
-# * distributed under the License is distributed on an "AS IS" BASIS,
-# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# * See the License for the specific language governing permissions and
-# * limitations under the License.
-# *-------------------------------------------------------------------------------
-# * For more information about the OpenAirInterface (OAI) Software Alliance:
-# *	  contact@openairinterface.org
-# */
-#---------------------------------------------------------------------
+"""
+Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The OpenAirInterface Software Alliance licenses this file to You under
+the OAI Public License, Version 1.1  (the "License"); you may not use this file
+except in compliance with the License.
+You may obtain a copy of the License at
+
+      http://www.openairinterface.org/?page_id=698
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+------------------------------------------------------------------------------
+For more information about the OpenAirInterface (OAI) Software Alliance:
+      contact@openairinterface.org
+"""
 
 import glob
 import os
 import re
 import sys
 import subprocess
+from pcap_check import *
 
 class HtmlReport():
 	def __init__(self):
@@ -693,24 +693,46 @@ class HtmlReport():
 		self.addImageRow('oai_smf')
 		self.addImageRow('oai_upf_vpp')
 		self.file.write('  </table>\n')
+
+		if os.path.isfile(cwd + '/archives/ci-upf-vpp-sanity.pcap'):
+			res1 = check_if_upf_registers_to_nrf('./archives/ci-upf-vpp-sanity.pcap')
+			res2 = check_pcfp_association('./archives/ci-upf-vpp-sanity.pcap')
+			self.file.write('  <br>\n')
+			self.file.write('  <table class="table-bordered" width = "90%" align = "center" border = 1>\n')
+			self.file.write('     <tr bgcolor = "#33CCFF" >\n')
+			self.file.write('       <th>Test Name</th>\n')
+			self.file.write('       <th>Test Status</th>\n')
+			self.file.write('       <th>Test Details</th>\n')
+			self.file.write('     </tr>\n')
+			self.file.write(self.addSectionRow('UPF registration to NRF'))
+			self.file.write(self.addDetailsRow('UPF Request', res1['upf_nrf_reg_req'], 'n/a'))
+			self.file.write(self.addDetailsRow('NRF Answer', res1['upf_nrf_reg_res'], 'UPF_FQDN   = ' + res1['upf_nrf_fqdn']))
+			self.file.write(self.addSectionRow('PFCP Association'))
+			self.file.write(self.addDetailsRow('SMF Request', res2['pfcp_ass_req'], 'PFCP IPv4  = ' + res2['pfcp_ass_ipv4']))
+			details  = 'FQDN       = ' + res2['pfcp_ass_upf_fqdn'] + '\n'
+			details += 'Entreprise = ' + res2['pfcp_ass_entreprise'] + '\n'
+			details += 'Build      = ' + res2['pfcp_ass_build_str']
+			self.file.write(self.addDetailsRow('UPF Answer', res2['pfcp_ass_res'], details))
+			self.file.write('  </table>\n')
+
 		self.file.write('  </div>\n')
 
 	def addImageRow(self, imageInfoPrefix):
 		cwd = os.getcwd()
 		if imageInfoPrefix == 'oai_amf':
-			containerName = 'oai-amf'
+			containerName = 'ci-oai-amf'
 			tagPattern = 'OAI_AMF_TAG'
 		if imageInfoPrefix == 'oai_smf':
-			containerName = 'oai-smf'
+			containerName = 'ci-oai-smf'
 			tagPattern = 'OAI_SMF_TAG'
 		if imageInfoPrefix == 'oai_nrf':
-			containerName = 'oai-nrf'
+			containerName = 'ci-oai-nrf'
 			tagPattern = 'OAI_NRF_TAG'
 		if imageInfoPrefix == 'oai_upf_vpp':
-			containerName = 'vpp-upf'
+			containerName = 'ci-vpp-upf'
 			tagPattern = 'OAI_UPF_VPP_TAG'
 		if imageInfoPrefix == 'mysql':
-			containerName = imageInfoPrefix
+			containerName = 'ci-mysql'
 			tagPattern = 'MYSQL_TAG'
 		if os.path.isfile(cwd + '/archives/' + imageInfoPrefix + '_image_info.log'):
 			usedTag = ''
@@ -742,6 +764,25 @@ class HtmlReport():
 			self.file.write('       <td>' + size + '</td>\n')
 			self.file.write('     </tr>\n')
 
+	def addSectionRow(self, sectionName):
+		sectionRow = ''
+		sectionRow += '     <tr bgcolor = "LightGray">\n'
+		sectionRow += '       <td align = "center" colspan = "3">' + sectionName + '</td>\n'
+		sectionRow += '     </tr>\n'
+		return sectionRow
+
+	def addDetailsRow(self, testName, status, details):
+		detailsRow = ''
+		detailsRow += '     <tr>\n'
+		detailsRow += '       <td>' + testName + '</td>\n'
+		if status:
+			detailsRow += '       <td bgcolor = "Green"><font color="white"><b>OK</b></font></td>\n'
+		else:
+			detailsRow += '       <td bgcolor = "Red"><font color="white"><b>KO</b></font></td>\n'
+		detailsRow += '       <td><pre>'+ details + '</td>\n'
+		detailsRow += '     </tr>\n'
+		return detailsRow
+
 	def testSummaryFooter(self):
 		self.file.write('  <br>\n')
 

ci-scripts/pcap_check.py

+"""
+Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The OpenAirInterface Software Alliance licenses this file to You under
+the OAI Public License, Version 1.1  (the "License"); you may not use this file
+except in compliance with the License.
+You may obtain a copy of the License at
+
+      http://www.openairinterface.org/?page_id=698
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+------------------------------------------------------------------------------
+For more information about the OpenAirInterface (OAI) Software Alliance:
+      contact@openairinterface.org
+"""
+
+import os
+import re
+import socket
+import struct
+import sys
+import subprocess
+import pyshark
+
+# IP addresses
+NRF_IP_ADDRESS = '192.168.71.130'
+SMF_IP_ADDRESS = '192.168.71.133'
+UPF_IP_ADDRESS = '192.168.71.134'
+UPF_N4_IP_ADDRESS = '192.168.71.202'
+
+# PGCP Constants
+PFCP_MSG_TYPE__SX_HEARTBEAT_REQUEST = '1'
+PFCP_MSG_TYPE__SX_HEARTBEAT_RESPONSE = '2'
+PFCP_MSG_TYPE__SX_ASSOCIATION_SETUP_REQUEST = '5'
+PFCP_MSG_TYPE__SX_ASSOCIATION_SETUP_RESPONSE = '6'
+PFCP_NODE_ID_TYPE__IPV4 = '0'
+PFCP_NODE_ID_TYPE__FQDN = '2'
+PFCP_CAUSE__SUCCESS = '1'
+
+def check_if_upf_registers_to_nrf(pcap_file):
+	res = {}
+	res['upf_nrf_reg_req'] = False
+	res['upf_nrf_reg_res'] = False
+	res['upf_nrf_fqdn'] = ''
+	try:
+		cap = {}
+		#cap = pyshark.FileCapture(pcap_file, keep_packets=True, display_filter="http2", use_json=True)
+		cap = pyshark.FileCapture(pcap_file, keep_packets=True, display_filter="http2")
+		cnt = 0
+		for pkt in cap:
+			if pkt is not None:
+				cnt += 1
+				if pkt.ip.src == UPF_IP_ADDRESS and pkt.ip.dst == NRF_IP_ADDRESS:
+					result = re.search('headers_method', str(pkt.http2.field_names))
+					if result is not None:
+						if pkt.http2.headers_method == 'PUT':
+							res['upf_nrf_reg_req'] = True
+				if pkt.ip.src == NRF_IP_ADDRESS and pkt.ip.dst == UPF_IP_ADDRESS:
+					status201 = False
+					upf_fqdn = False
+					result = re.search('header', str(pkt.http2.field_names))
+					if result is not None:
+						if pkt.http2.header == 'Header: :status: 201 Created':
+							status201 = True
+					result = re.search('json_value_string', str(pkt.http2.field_names))
+					if result is not None:
+						if pkt.http2.json_value_string == 'gw1.vppupf.node.5gcn.mnc95.mcc208.3gppnetwork.org':
+							upf_fqdn = True
+						res['upf_nrf_fqdn'] = pkt.http2.json_value_string
+					if status201 and upf_fqdn:
+						res['upf_nrf_reg_res'] = True
+		cap.close()
+	except Exception as e:
+		print(e.message, e.args) 
+		print('Could not open PCAP file')
+	return res
+
+def check_pcfp_association(pcap_file):
+	res = {}
+	res['pfcp_ass_req'] = False
+	res['pfcp_ass_res'] = False
+	res['pfcp_ass_ipv4'] = ''
+	res['pfcp_ass_upf_fqdn'] = ''
+	res['pfcp_ass_entreprise'] = ''
+	res['pfcp_ass_build_str'] = ''
+	try:
+		cap = {}
+		cap = pyshark.FileCapture(pcap_file, keep_packets=True, display_filter="pfcp")
+		cnt = 0
+		for pkt in cap:
+			if pkt is not None:
+				cnt += 1
+				if pkt.ip.src == SMF_IP_ADDRESS and pkt.ip.dst == UPF_N4_IP_ADDRESS:
+					if pkt.pfcp.msg_type == PFCP_MSG_TYPE__SX_ASSOCIATION_SETUP_REQUEST and pkt.pfcp.node_id_type == PFCP_NODE_ID_TYPE__IPV4:
+						if pkt.pfcp.node_id_ipv4 == SMF_IP_ADDRESS:
+							res['pfcp_ass_req'] = True
+							res['pfcp_ass_ipv4'] = pkt.pfcp.node_id_ipv4
+						else:
+							res['pfcp_ass_ipv4'] = pkt.pfcp.node_id_ipv4
+				if pkt.ip.src == UPF_N4_IP_ADDRESS and pkt.ip.dst == SMF_IP_ADDRESS:
+					if pkt.pfcp.msg_type == PFCP_MSG_TYPE__SX_ASSOCIATION_SETUP_RESPONSE and pkt.pfcp.node_id_type == PFCP_NODE_ID_TYPE__FQDN:
+						if pkt.pfcp.cause == PFCP_CAUSE__SUCCESS:
+							res['pfcp_ass_res'] = True
+						res['pfcp_ass_upf_fqdn'] = pkt.pfcp.node_id_fqdn
+						if pkt.pfcp.enterprise_id == '18681':
+							res['pfcp_ass_entreprise'] = 'Travelping GmbH'
+						res['pfcp_ass_build_str'] = pkt.pfcp.travelping_build_id_str
+		cap.close()
+	except Exception as e:
+		print(e.message, e.args) 
+		print('Could not open PCAP file')
+	return res

ci-scripts/sanityCheckDeployment.py

@@ -27,6 +27,7 @@ import shutil
 import subprocess
 import sys
 import time
+from pcap_check import *
 
 logging.basicConfig(
     level=logging.DEBUG,
@@ -90,6 +91,7 @@ def deployObject(service, tag, compose_file):
 
     if service == 'gnbsim-vpp':
         logging.info ('Not available for the moment')
+        time.sleep(10)
         return
 
     if not os.path.isdir('./archives'):
@@ -99,6 +101,7 @@ def deployObject(service, tag, compose_file):
     if not os.path.exists('./ci-' + compose_file):
         logging.debug('cp ./' + compose_file + ' ./ci-' + compose_file)
         shutil.copyfile('./' + compose_file, './ci-' + compose_file)
+        subprocess_run_w_echo('sed -i -e \'s@container_name: "@container_name: "ci-@\' ./ci-' + compose_file)
     expect_healthy = 0
     nb_healthy = 0
     cnt = 0
@@ -150,6 +153,9 @@ def deployObject(service, tag, compose_file):
                 nb_healthy += 1
     if nb_healthy == expect_healthy:
         logging.info('OK')
+        # Adding a tempo for VPP-UPF to be fully ready
+        if service == 'vpp-upf':
+            time.sleep(10)
     elif cnt > 50:
         logging.error('KO')
         sys.exit(-1)
@@ -162,7 +168,7 @@ def undeployObject(service, compose_file):
         return
 
     os.chdir('docker-compose')
-    if not os.path.exists('./ci-' + compose_file):
+    if os.path.exists('./ci-' + compose_file):
         subprocess_run_w_echo('docker-compose -p vpp-sanity -f ./ci-' + compose_file + ' down')
     else:
         subprocess_run_w_echo('docker-compose -p vpp-sanity -f ./' + compose_file + ' down')
@@ -177,8 +183,24 @@ def checkDeployment(service, compose_file):
         subprocess_run_w_echo('mkdir -p ./archives')
 
     if os.path.exists('/tmp/ci-upf-vpp-sanity.pcap'):
-        subprocess_run_w_echo('cp /tmp/ci-upf-vpp-sanity.pcap archives')
         subprocess_run_w_echo('sudo chmod 666 /tmp/ci-upf-vpp-sanity.pcap')
+        subprocess_run_w_echo('cp /tmp/ci-upf-vpp-sanity.pcap archives')
+
+    upf_register = False
+    pfcp_association = False
+    if os.path.exists('./archives/ci-upf-vpp-sanity.pcap'):
+        res1 = check_if_upf_registers_to_nrf('./archives/ci-upf-vpp-sanity.pcap')
+        if res1['upf_nrf_reg_req'] and res1['upf_nrf_reg_res']:
+            logging.debug('UPF did register at NRF')
+            upf_register = True
+        else:
+            logging.error('UPF did NOT register at NRF')
+        res2 = check_pcfp_association('./archives/ci-upf-vpp-sanity.pcap')
+        if res2['pfcp_ass_req'] and res2['pfcp_ass_res']:
+            logging.debug('PCFP association b/w SMF and UPF was successful')
+            pfcp_association = True
+        else:
+            logging.error('PCFP association b/w SMF and UPF failed')
 
     os.chdir('docker-compose')
     if not os.path.exists('./ci-' + compose_file):
@@ -186,12 +208,17 @@ def checkDeployment(service, compose_file):
 
     cmd = 'docker-compose -p vpp-sanity -f ./ci-' + compose_file + ' ps -a'
     res = subprocess.check_output(cmd, shell=True, universal_newlines=True)
+    healthy_cnt = 0
     for line in res.split('\n'):
         result = re.search('Up.*healthy', line)
         if result is not None:
             subprocess_run_w_echo('docker logs ' + line.split(' ')[0] + ' > ../archives/' + line.split(' ')[0] + '.log 2>&1')
+            healthy_cnt += 1
 
-    subprocess_run_w_echo('echo "SANITY-CHECK-DEPLOYMENT: OK" > ../archives/deployment_status.log')
+    if upf_register and (healthy_cnt == 5):
+        subprocess_run_w_echo('echo "SANITY-CHECK-DEPLOYMENT: OK" > ../archives/deployment_status.log')
+    else:
+        subprocess_run_w_echo('echo "SANITY-CHECK-DEPLOYMENT: KO" > ../archives/deployment_status.log')
 
 def subprocess_run_w_echo(cmd):
     logging.debug(cmd)

docker-compose/docker-compose.yml

@@ -156,7 +156,7 @@ services:
     vpp-upf:
         image: oai-upf-vpp:latest
         privileged: true
-        container_name: vpp-upf
+        container_name: "vpp-upf"
         environment:
             - NWI_N3=access.oai.org
             - NWI_N6=core.oai.org
@@ -197,7 +197,7 @@ services:
     oai-ext-dn:
         image: ubuntu:bionic
         privileged: true
-        container_name: oai-ext-dn
+        container_name: "oai-ext-dn"
         entrypoint: /bin/bash -c \
               "apt update; apt install -y iptables iproute2 iputils-ping;"\
               "iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE;"\