feat(ci): adding sanity check deployment, no proper verification

Signed-off-by: Raphael Defosseux <raphael.defosseux@eurecom.fr>

ci-scripts/Jenkinsfile-GitLab-Docker

@@ -236,17 +236,21 @@ pipeline {
           sh 'python3 ./ci-scripts/sanityCheckDeployment.py --mode Deploy --service vpp-upf --upf_tag ' + upf_tag
           // SMF is dependant on AMF and EXT-DN: they will be deploy as well
           sh 'python3 ./ci-scripts/sanityCheckDeployment.py --mode Deploy --service oai-smf'
+          // GNB SIM as RAN emulator
+          sh 'python3 ./ci-scripts/sanityCheckDeployment.py --mode Deploy --docker_compose docker-compose-gnbsim.yaml --service gnbsim-vpp'
 
           // Check if deployment is OK
           sh 'python3 ./ci-scripts/sanityCheckDeployment.py --mode Check --service all'
 
           // Undeploy once OK
+          sh 'python3 ./ci-scripts/sanityCheckDeployment.py --mode UnDeploy --docker_compose docker-compose-gnbsim.yaml --service gnbsim-vpp'
           sh 'python3 ./ci-scripts/sanityCheckDeployment.py --mode UnDeploy --service all'
         }
       }
       post {
         unsuccessful {
-          sh 'python3 ./ci-scripts/sanityCheckDeployment.py --mode UnDeploy --service all'
+          sh 'python3 ./ci-scripts/sanityCheckDeployment.py --mode Check --service all || true'
+          sh 'python3 ./ci-scripts/sanityCheckDeployment.py --mode UnDeploy --service all || true'
         }
       }
     }

ci-scripts/generateHtmlReport.py

@@ -656,9 +656,91 @@ class HtmlReport():
 
 	def testSummaryHeader(self):
 		self.file.write('  <h2>Test Summary</h2>\n')
+		cwd = os.getcwd()
+		if os.path.isfile(cwd + '/archives/deployment_status.log'):
+			cmd = 'egrep -c "DEPLOYMENT: OK" archives/deployment_status.log || true'
+			status = False
+			ret = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE, encoding='utf-8')
+			if ret.stdout is not None:
+				status = True
+			if status:
+				self.file.write('  <div class="alert alert-success">\n')
+				self.file.write('	  <strong>Successful Sanity Check Deployment! <span class="glyphicon glyphicon-warning-sign"></span></strong>\n')
+				self.file.write('  </div>\n')
+			else:
+				self.file.write('  <div class="alert alert-danger">\n')
+				self.file.write('	  <strong>Failed Sanity Check Deployment! <span class="glyphicon glyphicon-warning-sign"></span></strong>\n')
+				self.file.write('  </div>\n')
+		else:
 			self.file.write('  <div class="alert alert-warning">\n')
-		self.file.write('	  <strong>Not performed yet. <span class="glyphicon glyphicon-warning-sign"></span></strong>\n')
+			self.file.write('	  <strong>Not performed. <span class="glyphicon glyphicon-warning-sign"></span></strong>\n')
 			self.file.write('  </div>\n')
+		self.file.write('  <br>\n')
+		self.file.write('  <button data-toggle="collapse" data-target="#deployment-details">More details on Deployment</button>\n')
+		self.file.write('  <br>\n')
+		self.file.write('  <div id="deployment-details" class="collapse">\n')
+		self.file.write('  <br>\n')
+		self.file.write('  <table class="table-bordered" width = "80%" align = "center" border = 1>\n')
+		self.file.write('     <tr bgcolor = "#33CCFF" >\n')
+		self.file.write('       <th>Container Name</th>\n')
+		self.file.write('       <th>Used Image Tag</th>\n')
+		self.file.write('       <th>Image Creation Date</th>\n')
+		self.file.write('       <th>Used Image Size</th>\n')
+		self.file.write('     </tr>\n')
+		self.addImageRow('mysql')
+		self.addImageRow('oai_nrf')
+		self.addImageRow('oai_amf')
+		self.addImageRow('oai_smf')
+		self.addImageRow('oai_upf_vpp')
+		self.file.write('  </table>\n')
+		self.file.write('  </div>\n')
+
+	def addImageRow(self, imageInfoPrefix):
+		cwd = os.getcwd()
+		if imageInfoPrefix == 'oai_amf':
+			containerName = 'oai-amf'
+			tagPattern = 'OAI_AMF_TAG'
+		if imageInfoPrefix == 'oai_smf':
+			containerName = 'oai-smf'
+			tagPattern = 'OAI_SMF_TAG'
+		if imageInfoPrefix == 'oai_nrf':
+			containerName = 'oai-nrf'
+			tagPattern = 'OAI_NRF_TAG'
+		if imageInfoPrefix == 'oai_upf_vpp':
+			containerName = 'vpp-upf'
+			tagPattern = 'OAI_UPF_VPP_TAG'
+		if imageInfoPrefix == 'mysql':
+			containerName = imageInfoPrefix
+			tagPattern = 'MYSQL_TAG'
+		if os.path.isfile(cwd + '/archives/' + imageInfoPrefix + '_image_info.log'):
+			usedTag = ''
+			createDate = ''
+			size = ''
+			with open(cwd + '/archives/' + imageInfoPrefix + '_image_info.log') as imageLog:
+				for line in imageLog:
+					line = line.strip()
+					result = re.search(tagPattern + ': (?P<tag>[a-zA-Z0-9\.\-\_:]+)', line)
+					if result is not None:
+						usedTag = result.group('tag')
+					result = re.search('Date = (?P<date>[a-zA-Z0-9\-\_:]+)', line)
+					if result is not None:
+						createDate = result.group('date')
+					result = re.search('Size = (?P<size>[0-9]+) bytes', line)
+					if result is not None:
+						sizeInt = int(result.group('size'))
+						if sizeInt < 1000000:
+							sizeInt = int(sizeInt / 1000)
+							size = str(sizeInt) + ' kB'
+						else:
+							sizeInt = int(sizeInt / 1000000)
+							size = str(sizeInt) + ' MB'
+			imageLog.close()
+			self.file.write('     <tr>\n')
+			self.file.write('       <td>' + containerName + '</td>\n')
+			self.file.write('       <td>' + usedTag + '</td>\n')
+			self.file.write('       <td>' + createDate + '</td>\n')
+			self.file.write('       <td>' + size + '</td>\n')
+			self.file.write('     </tr>\n')
 
 	def testSummaryFooter(self):
 		self.file.write('  <br>\n')

ci-scripts/sanityCheckDeployment.py

@@ -23,8 +23,10 @@ import argparse
 import logging
 import os
 import re
-import sys
+import shutil
 import subprocess
+import sys
+import time
 
 logging.basicConfig(
     level=logging.DEBUG,
@@ -36,12 +38,13 @@ def main() -> None:
     args = _parse_args()
 
     if args.mode == 'Deploy':
-        deployObject(args.service, args.upf_tag)
+        deployObject(args.service, args.upf_tag, args.docker_compose)
 
     if args.mode == 'Check':
-        logging.info ('Performing Check')
+        checkDeployment(args.service, args.docker_compose)
+
     if args.mode == 'UnDeploy':
-        logging.info ('Un-Deploying ' + args.service)
+        undeployObject(args.service, args.docker_compose)
 
 def _parse_args() -> argparse.Namespace:
     """Parse the command line args
@@ -82,10 +85,103 @@ def _parse_args() -> argparse.Namespace:
     )
     return parser.parse_args()
 
-def deployObject(service, tag):
+def deployObject(service, tag, compose_file):
     logging.info ('Deploying ' + service)
+
+    if service == 'gnbsim-vpp':
+        logging.info ('Not available for the moment')
+        return
+
+    if not os.path.isdir('./archives'):
+        subprocess_run_w_echo('mkdir -p ./archives')
+
+    os.chdir('docker-compose')
+    if not os.path.exists('./ci-' + compose_file):
+        logging.debug('cp ./' + compose_file + ' ./ci-' + compose_file)
+        shutil.copyfile('./' + compose_file, './ci-' + compose_file)
+    expect_healthy = 0
+    nb_healthy = 0
+    cnt = 0
+    # mysql does not require image tag update
+    # Also starting a capture on the public docker network
+    if service == 'mysql':
+        subprocess_run_w_echo('echo "MYSQL_TAG: mysql:5.7" > ../archives/mysql_image_info.log')
+        subprocess_run_w_echo('docker image inspect --format="Size = {{.Size}} bytes" mysql:5.7 >> ../archives/mysql_image_info.log')
+        subprocess_run_w_echo('docker image inspect --format="Date = {{.Created}}" mysql:5.7 >> ../archives/mysql_image_info.log')
+        subprocess_run_w_echo('docker-compose -p vpp-sanity -f ./ci-' + compose_file + ' up -d ' + service)
+        expect_healthy = 1
+        subprocess_run_w_echo('sudo nohup tshark -i cn5g-public -f "not arp and not port 53 and not host archive.ubuntu.com and not host security.ubuntu.com" -w /tmp/ci-upf-vpp-sanity.pcap > /dev/null 2>&1 &')
+    # We might need tag manipulations
+    if service == 'oai-nrf':
+        subprocess_run_w_echo('echo "OAI_NRF_TAG: oai-nrf:develop" > ../archives/oai_nrf_image_info.log')
+        subprocess_run_w_echo('docker image inspect --format="Size = {{.Size}} bytes" oai-nrf:develop >> ../archives/oai_nrf_image_info.log')
+        subprocess_run_w_echo('docker image inspect --format="Date = {{.Created}}" oai-nrf:develop >> ../archives/oai_nrf_image_info.log')
+        subprocess_run_w_echo('sed -i -e "s#oai-nrf:latest#oai-nrf:develop#" ./ci-' + compose_file)
+        subprocess_run_w_echo('docker-compose -p vpp-sanity -f ./ci-' + compose_file + ' up -d ' + service)
+        expect_healthy = 2
+    if service == 'vpp-upf':
+        subprocess_run_w_echo('echo "OAI_UPF_VPP_TAG: oai-upf-vpp:' + tag + '" > ../archives/oai_upf_vpp_image_info.log')
+        subprocess_run_w_echo('docker image inspect --format="Size = {{.Size}} bytes" oai-upf-vpp:' + tag + ' >> ../archives/oai_upf_vpp_image_info.log')
+        subprocess_run_w_echo('docker image inspect --format="Date = {{.Created}}" oai-upf-vpp:' + tag + ' >> ../archives/oai_upf_vpp_image_info.log')
+        subprocess_run_w_echo('sed -i -e "s#oai-upf-vpp:latest#oai-upf-vpp:' + tag + '#" ./ci-' + compose_file)
+        subprocess_run_w_echo('docker-compose -p vpp-sanity -f ./ci-' + compose_file + ' up -d ' + service)
+        expect_healthy = 3
+    if service == 'oai-smf':
+        subprocess_run_w_echo('echo "OAI_AMF_TAG: oai-amf:develop" > ../archives/oai_amf_image_info.log')
+        subprocess_run_w_echo('docker image inspect --format="Size = {{.Size}} bytes" oai-amf:develop >> ../archives/oai_amf_image_info.log')
+        subprocess_run_w_echo('docker image inspect --format="Date = {{.Created}}" oai-amf:develop >> ../archives/oai_amf_image_info.log')
+        subprocess_run_w_echo('echo "OAI_SMF_TAG: oai-smf:develop" > ../archives/oai_smf_image_info.log')
+        subprocess_run_w_echo('docker image inspect --format="Size = {{.Size}} bytes" oai-smf:develop >> ../archives/oai_smf_image_info.log')
+        subprocess_run_w_echo('docker image inspect --format="Date = {{.Created}}" oai-smf:develop >> ../archives/oai_smf_image_info.log')
+        subprocess_run_w_echo('sed -i -e "s#oai-amf:latest#oai-amf:develop#" -e "s#oai-smf:latest#oai-smf:develop#" ./ci-' + compose_file)
+        subprocess_run_w_echo('docker-compose -p vpp-sanity -f ./ci-' + compose_file + ' up -d ' + service)
+        expect_healthy = 5
+
+    # Counting the number of healthy containers
+    cmd = 'docker-compose -p vpp-sanity -f ./ci-' + compose_file + ' ps -a'
+    while (nb_healthy != expect_healthy) or (cnt > 50):
+        time.sleep(5)
+        cnt += 1
+        res = subprocess.check_output(cmd, shell=True, universal_newlines=True)
+        nb_healthy = 0
+        for line in res.split('\n'):
+            result = re.search('Up.*healthy', line)
+            if result is not None:
+                nb_healthy += 1
+    if nb_healthy == expect_healthy:
+        logging.info('OK')
+    elif cnt > 50:
+        logging.error('KO')
+        sys.exit(-1)
+
+def undeployObject(service, compose_file):
+    logging.info ('Un-Deploying ' + service)
+
+    if service == 'gnbsim-vpp':
+        logging.info ('Not available for the moment')
+        return
+
     os.chdir('docker-compose')
-    subprocess_run_w_echo ('ls')
+    subprocess_run_w_echo('docker-compose -p vpp-sanity -f ./ci-' + compose_file + ' down')
+
+def checkDeployment(service, compose_file):
+    logging.info ('Performing Check')
+    subprocess_run_w_echo('sudo chmod 666 /tmp/ci-upf-vpp-sanity.pcap')
+    if not os.path.isdir('./archives'):
+        subprocess_run_w_echo('mkdir -p ./archives')
+
+    if os.path.exists('/tmp/ci-upf-vpp-sanity.pcap'):
+        subprocess_run_w_echo('cp /tmp/ci-upf-vpp-sanity.pcap archives')
+
+    os.chdir('docker-compose')
+    cmd = 'docker-compose -p vpp-sanity -f ./ci-' + compose_file + ' ps -a'
+    res = subprocess.check_output(cmd, shell=True, universal_newlines=True)
+    for line in res.split('\n'):
+        result = re.search('Up.*healthy', line)
+        if result is not None:
+            subprocess_run_w_echo('docker logs ' + line.split(' ')[0] + ' > ../archives/' + line.split(' ')[0] + '.log 2>&1')
+
+    subprocess_run_w_echo('echo "SANITY-CHECK-DEPLOYMENT: OK" > ../archives/deployment_status.log')
 
 def subprocess_run_w_echo(cmd):
     logging.debug(cmd)