CI: initial build pipeline

Signed-off-by: Raphael Defosseux <raphael.defosseux@eurecom.fr>

ci-scripts/Jenkinsfile-GitLab-Docker

+#!/bin/groovy
+/*
+ * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The OpenAirInterface Software Alliance licenses this file to You under
+ * the OAI Public License, Version 1.1  (the "License"); you may not use this file
+ * except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.openairinterface.org/?page_id=698
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *-------------------------------------------------------------------------------
+ * For more information about the OpenAirInterface (OAI) Software Alliance:
+ *      contact@openairinterface.org
+ */
+
+//-------------------------------------------------------------------------------
+
+// Location of the CN executor node
+def cn_ci_host = params.Host_CN_CI_Server
+
+// for lock
+def cn_ci_resource = params.DockerContainers
+
+// Location of the Remote Ubuntu18 CN executor
+def rem_u18_host_flag = false
+def rem_u18_host = ""
+def rem_u18_host_user = ""
+
+// Location of the Remote RHEL CN executor
+def rem_rhel_host_flag = false
+def rem_rhel_host = ""
+def rem_rhel_host_user = ""
+def rem_rhel8_resource = params.PodmanContainers
+
+// Tags/Branches to use
+def upf_tag = "develop"
+def upf_branch = "develop"
+
+// Merge Request Link
+def gitlabMergeRequestLink
+
+//-------------------------------------------------------------------------------
+// Pipeline start
+pipeline {
+  agent {
+    label cn_ci_host
+  }
+  options {
+    disableConcurrentBuilds()
+    timestamps()
+    ansiColor('xterm')
+    lock(cn_ci_resource)
+    gitLabConnection('OAI GitLab')
+    gitlabBuilds(builds: [
+      "Build RHEL7 UPF-VPP Image",
+      "Build U18 UPF-VPP Image"
+      //"Test with DsTester"
+    ])
+
+  }
+
+  stages {
+    stage ('Verify Parameters') {
+      steps {
+        script {
+          echo '\u2705 \u001B[32mVerify Parameters\u001B[0m'
+
+          JOB_TIMESTAMP = sh returnStdout: true, script: 'date --utc --rfc-3339=seconds | sed -e "s#+00:00##"'
+          JOB_TIMESTAMP = JOB_TIMESTAMP.trim()
+
+          if (params.Host_CN_CI_2nd_Server_Flag != null) {
+            rem_u18_host_flag = params.Host_CN_CI_2nd_Server_Flag
+            if (rem_u18_host_flag) {
+              rem_u18_host = params.Host_CN_CI_2nd_Server
+              rem_u18_host_user = params.Host_CN_CI_2nd_Server_Login
+              echo "1st Node   is ${NODE_NAME}"
+              echo "2nd Node   is ${rem_u18_host}"
+            } else {
+              echo "U18 Node   is ${NODE_NAME}"
+            }
+          } else {
+            echo "Node       is ${NODE_NAME}"
+          }
+          if (params.Remote_RHEL_Server_Flag != null) {
+            rem_rhel_host_flag = params.Remote_RHEL_Server_Flag
+            if (rem_rhel_host_flag) {
+              rem_rhel_host = params.Remote_RHEL_Server_Name
+              rem_rhel_host_user = params.Remote_RHEL_Server_Login
+              echo "RHEL Node  is ${rem_rhel_host}"
+            }
+          }
+          echo "Git URL    is ${GIT_URL}"
+        }
+      }
+    }
+    stage ('Prepare Source Code') {
+      steps {
+        script {
+          sh "git clean -x -d -f > /dev/null 2>&1"
+          if ("MERGE".equals(env.gitlabActionType)) {
+            // Need a public repo
+            //gitlabMergeRequestLink = sh "curl --silent \"https://gitlab.eurecom.fr/api/v4/projects/oai%2Fcn5g%2Foai-cn5g-upf-vpp/merge_requests/${env.gitlabMergeRequestIid}\" | jq .web_url | sed 's#\"##g'"
+            echo "========= THIS IS A MERGE REQUEST =========="
+            echo "MR ID       is ${env.gitlabMergeRequestIid}"
+            //echo "MR LINK     is ${gitlabMergeRequestLink}"
+            echo "MR TITLE    is ${env.gitlabMergeRequestTitle}"
+            gitCommitAuthorEmailAddr = env.gitlabUserEmail
+            echo "GitLab Usermail is ${gitCommitAuthorEmailAddr}"
+            // Need a public repo
+            //sh "git fetch --prune --unshallow"
+            shortenShaOne = sh returnStdout: true, script: 'git log -1 --pretty=format:"%h" ' + env.gitlabMergeRequestLastCommit
+            shortenShaOne = shortenShaOne.trim()
+            sh "./ci-scripts/doGitLabMerge.sh --src-branch ${env.gitlabSourceBranch} --src-commit ${env.gitlabMergeRequestLastCommit} --target-branch ${env.gitlabTargetBranch} --target-commit ${GIT_COMMIT}"
+            sh "sleep 10"
+            upf_tag = "ci-tmp"
+            rhel_upf_tag = 'ci-tmp-pr-' + env.gitlabMergeRequestIid + '-' + shortenShaOne
+            upf_branch = env.gitlabSourceBranch
+            echo "MR TAG      is ${rhel_upf_tag}"
+          } else {
+            echo "======== THIS IS A PUSH REQUEST ========"
+            echo "Git Branch      is ${GIT_BRANCH}"
+            echo "Git Commit      is ${GIT_COMMIT}"
+            gitCommitAuthorEmailAddr = sh returnStdout: true, script: 'git log -n1 --pretty=format:%ae ${GIT_COMMIT}'
+            gitCommitAuthorEmailAddr = gitCommitAuthorEmailAddr.trim()
+            echo "GitLab Usermail is ${gitCommitAuthorEmailAddr}"
+            sh "git log -n1 --pretty=format:\"%s\" > .git/CI_COMMIT_MSG"
+          }
+          sh "tar -cjhf /tmp/openair-upf.tar.bz2 ."
+          sh "mv /tmp/openair-upf.tar.bz2 ."
+          copyTo2ndServer('openair-upf.tar.bz2', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
+          copyTo2ndServer('openair-upf.tar.bz2', rem_rhel_host_flag, rem_rhel_host_user, rem_rhel_host)
+        }
+      }
+      post {
+        failure {
+          script {
+            def message = "OAI " + JOB_NAME + " build (" + BUILD_ID + "): Merge Conflicts -- Cannot perform CI"
+            addGitLabMRComment comment: message
+            currentBuild.result = 'FAILURE'
+          }
+        }
+      }
+    }
+    stage('Build Core Network Function') {
+      parallel {
+        stage ('Build U18 UPF-VPP Image') {
+          steps {
+            script {
+              gitlabCommitStatus(name: "Build U18 UPF-VPP Image") {
+                myShCmd('docker image prune --force', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
+                // Currently this pipeline only runs for pushes to `develop` branch
+                // First clean image registry
+                myShCmd('docker image rm oai-upf:' + upf_tag + ' || true', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
+                myShCmd('docker build --no-cache --target oai-upf-vpp --tag oai-upf-vpp:' + upf_tag + ' --file docker/Dockerfile.upf-vpp.ubuntu18 --build-arg NEEDED_GIT_PROXY="http://proxy.eurecom.fr:8080" . > archives/upf_docker_image_build.log 2>&1', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
+                myShCmd('docker image ls >> archives/upf_docker_image_build.log', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
+              }
+            }
+          }
+          post {
+            always {
+              script {
+                copyFrom2ndServer('archives/upf_docker_image_build.log', 'archives', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
+              }
+            }
+            success {
+              sh "echo 'OAI-UPF-VPP DOCKER IMAGE BUILD: OK' >> archives/upf_docker_image_build.log"
+            }
+            unsuccessful {
+              sh "echo 'OAI-UPF-VPP DOCKER IMAGE BUILD: KO' >> archives/upf_docker_image_build.log"
+            }
+          }
+        }
+        stage ('Build RHEL7 UPF-VPP Image') {
+          when { expression {rem_rhel_host_flag} }
+          steps {
+            lock (rem_rhel8_resource) {
+              script {
+                gitlabCommitStatus(name: "Build RHEL7 UPF-VPP Image") {
+                  myShCmd('sudo podman image rm oai-upf-vpp:' + upf_tag + ' || true', rem_rhel_host_flag, rem_rhel_host_user, rem_rhel_host)
+                  myShCmd('sudo podman image prune --force', rem_rhel_host_flag, rem_rhel_host_user, rem_rhel_host)
+                  // Copy the RHEL Host certificates for building
+                  myShCmd('mkdir -p tmp/ca tmp/entitlement', rem_rhel_host_flag, rem_rhel_host_user, rem_rhel_host)
+                  myShCmd('cp /etc/pki/entitlement/*pem tmp/entitlement', rem_rhel_host_flag, rem_rhel_host_user, rem_rhel_host)
+                  myShCmd('sudo cp /etc/rhsm/ca/redhat-uep.pem tmp/ca', rem_rhel_host_flag, rem_rhel_host_user, rem_rhel_host)
+                  myShCmd('sudo podman build --no-cache --target oai-upf-vpp --tag oai-upf-vpp:' + upf_tag + ' --file docker/Dockerfile.upf-vpp.rhel7 --build-arg NEEDED_GIT_PROXY="http://proxy.eurecom.fr:8080" . > archives/upf_podman_image_build.log 2>&1', rem_rhel_host_flag, rem_rhel_host_user, rem_rhel_host)
+                  myShCmd('sudo podman image ls >> archives/upf_podman_image_build.log', rem_rhel_host_flag, rem_rhel_host_user, rem_rhel_host)
+                  if ("MERGE".equals(env.gitlabActionType)) {
+                    myShCmd('sudo podman image tag oai-upf-vpp:' + upf_tag + ' oai-upf-vpp:' + rhel_upf_tag, rem_rhel_host_flag, rem_rhel_host_user, rem_rhel_host)
+                  }
+                }
+              }
+            }
+          }
+          post {
+            always {
+              script {
+                copyFrom2ndServer('archives/upf_podman_image_build.log', 'archives', rem_rhel_host_flag, rem_rhel_host_user, rem_rhel_host)
+              }
+            }
+            success {
+              sh "echo 'OAI-UPF-VPP PODMAN RHEL7 IMAGE BUILD: OK' >> archives/upf_podman_image_build.log"
+            }
+            unsuccessful {
+              sh "echo 'OAI-UPF-VPP PODMAN RHEL7 IMAGE BUILD: KO' >> archives/upf_podman_image_build.log"
+            }
+          }
+        }
+
+      }
+    }
+  }
+  post {
+    success {
+      script {
+        if ("MERGE".equals(env.gitlabActionType)) {
+          def message = "OAI " + JOB_NAME + " build (" + BUILD_ID + "): passed (" + BUILD_URL + ")"
+          echo "This is a MERGE event"
+          addGitLabMRComment comment: message
+        }
+      }
+    }
+    unsuccessful {
+      script {
+        if ("MERGE".equals(env.gitlabActionType)) {
+          def message = "OAI " + JOB_NAME + " build (" + BUILD_ID + "): failed (" + BUILD_URL + ")"
+          echo "This is a MERGE event"
+          addGitLabMRComment comment: message
+        }
+      }
+    }
+    cleanup {
+      script {
+        // Removing temporary / intermediate images
+        try {
+          if ("MERGE".equals(env.gitlabActionType)) {
+            myShCmd('docker image rm --force oai-upf:ci-tmp', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
+          }
+        } catch (Exception e) {
+          echo "We failed to delete the OAI-UPF-VPP temp image"
+        }
+        try {
+          if ("MERGE".equals(env.gitlabActionType)) {
+            myShCmd('sudo podman image rm oai-upf:ci-tmp', rem_rhel_host_flag, rem_rhel_host_user, rem_rhel_host)
+          }
+        } catch (Exception e) {
+          echo "We failed to delete the OAI-UPF-VPP temp image"
+        }
+        try {
+          myShCmd('docker image prune --force', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
+        } catch (Exception e) {
+          echo "We failed to prune all unneeded intermediate images"
+        }
+        if (rem_rhel_host_flag) {
+          try {
+            myShCmd('sudo podman image prune --force', rem_rhel_host_flag, rem_rhel_host_user, rem_rhel_host)
+          } catch (Exception e) {
+            echo "We failed to prune all unneeded intermediate images"
+          }
+        }
+
+        // Zipping all archived log files
+        sh "zip -r -qq docker_logs.zip archives"
+        if (fileExists('docker_logs.zip')) {
+          archiveArtifacts artifacts: 'docker_logs.zip'
+        }
+
+        // Generating the HTML report
+//        if ("MERGE".equals(env.gitlabActionType)) {
+//          sh "python3 ci-scripts/generateHtmlReport.py --job_name=${JOB_NAME} --job_id=${BUILD_ID} --job_url=${BUILD_URL} --git_url=${GIT_URL} --git_src_branch=${env.gitlabSourceBranch} --git_src_commit=${env.gitlabMergeRequestLastCommit} --git_pull_request=True --git_target_branch=${env.gitlabTargetBranch} --git_target_commit=${GIT_COMMIT}"
+//          if (fileExists('test_results_oai_upf.html')) {
+//            sh "sed -i -e 's#TEMPLATE_MERGE_REQUEST_LINK#${gitlabMergeRequestLink}#g' test_results_oai_upf.html"
+//            sh "sed -i -e 's#TEMPLATE_MERGE_REQUEST_TEMPLATE#${env.gitlabMergeRequestTitle}#' test_results_oai_upf.html"
+//          }
+//        } else {
+//          sh "python3 ci-scripts/generateHtmlReport.py --job_name=${JOB_NAME} --job_id=${BUILD_ID} --job_url=${BUILD_URL} --git_url=${GIT_URL} --git_src_branch=${GIT_BRANCH} --git_src_commit=${GIT_COMMIT}"
+//        }
+//        sh "sed -i -e 's#TEMPLATE_TIME#${JOB_TIMESTAMP}#' test_results_oai_upf.html"
+        if (fileExists('test_results_oai_upf.html')) {
+          sh "sed -i -e 's#TEMPLATE_TIME#${JOB_TIMESTAMP}#' test_results_oai_upf.html"
+          archiveArtifacts artifacts: 'test_results_oai_upf.html'
+        }
+
+        // Sending email to commiter
+        if (params.sendToCommitterEmail != null) {
+          if (params.sendToCommitterEmail) {
+            emailext attachmentsPattern: '*results*.html',
+              body: '''Hi,
+
+
+Here are attached HTML report files for $PROJECT_NAME - Build # $BUILD_NUMBER - $BUILD_STATUS!
+
+Regards,
+OAI CI Team''',
+              replyTo: 'no-reply@openairinterface.org',
+              subject: '$PROJECT_NAME - Build # $BUILD_NUMBER - $BUILD_STATUS!',
+              to: gitCommitAuthorEmailAddr
+          }
+        }
+      }
+    }
+  }
+}
+
+def copyTo2ndServer(filename, flag, user, host) {
+  if (flag) {
+    if ("openair-upf.tar.bz2".equals(filename)) {
+      sh "ssh ${user}@${host} 'rm -rf /tmp/CI-CN-UPF-VPP'"
+      sh "ssh ${user}@${host} 'mkdir -p /tmp/CI-CN-UPF-VPP'"
+    }
+    sh "scp ${filename} ${user}@${host}:/tmp/CI-CN-UPF-VPP"
+    if ("openair-upf.tar.bz2".equals(filename)) {
+      sh "ssh ${user}@${host} 'cd /tmp/CI-CN-UPF-VPP && tar -xjf ${filename}'"
+      sh "ssh ${user}@${host} 'mkdir -p /tmp/CI-CN-UPF-VPP/archives'"
+      sh "ssh ${user}@${host} 'rm -Rf /tmp/CI-CN-UPF-VPP/${filename}'"
+    }
+  }
+}
+
+def copyFrom2ndServer(filename, target, flag, user, host) {
+  if (flag) {
+    sh "scp ${user}@${host}:/tmp/CI-CN-UPF-VPP/${filename} ${target}"
+  }
+}
+
+def myShCmd(cmd, flag, user, host) {
+  if (flag) {
+    sh "ssh -t -t ${user}@${host} 'cd /tmp/CI-CN-UPF-VPP && ${cmd}'"
+  } else {
+    sh "${cmd}"
+  }
+}
+
+def myShRetCmd(cmd, flag, user, host) {
+  if (flag) {
+    ret = sh returnStdout: true, script: "ssh -t -t ${user}@${host} 'cd /tmp/CI-CN-UPF-VPP && ${cmd}'"
+  } else {
+    ret = sh returnStdout: true, script: "${cmd}"
+  }
+  ret = ret.trim()
+  return ret
+}

ci-scripts/doGitLabMerge.sh

+#!/bin/bash
+#/*
+# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
+# * contributor license agreements.  See the NOTICE file distributed with
+# * this work for additional information regarding copyright ownership.
+# * The OpenAirInterface Software Alliance licenses this file to You under
+# * the OAI Public License, Version 1.1  (the "License"); you may not use this file
+# * except in compliance with the License.
+# * You may obtain a copy of the License at
+# *
+# *      http://www.openairinterface.org/?page_id=698
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# *-------------------------------------------------------------------------------
+# * For more information about the OpenAirInterface (OAI) Software Alliance:
+# *      contact@openairinterface.org
+# */
+
+function usage {
+    echo "OAI GitLab merge request applying script"
+    echo "   Original Author: Raphael Defosseux"
+    echo ""
+    echo "Usage:"
+    echo "------"
+    echo ""
+    echo "    doGitLabMerge.sh [OPTIONS] [MANDATORY_OPTIONS]"
+    echo ""
+    echo "Mandatory Options:"
+    echo "------------------"
+    echo ""
+    echo "    --src-branch #### OR -sb ####"
+    echo "    Specify the source branch of the merge request."
+    echo ""
+    echo "    --src-commit #### OR -sc ####"
+    echo "    Specify the source commit ID (SHA-1) of the merge request."
+    echo ""
+    echo "    --target-branch #### OR -tb ####"
+    echo "    Specify the target branch of the merge request (usually develop)."
+    echo ""
+    echo "    --target-commit #### OR -tc ####"
+    echo "    Specify the target commit ID (SHA-1) of the merge request."
+    echo ""
+    echo "Options:"
+    echo "--------"
+    echo "    --help OR -h"
+    echo "    Print this help message."
+    echo ""
+}
+
+if [ $# -ne 8 ] && [ $# -ne 1 ]
+then
+    echo "Syntax Error: not the correct number of arguments"
+    echo ""
+    usage
+    exit 1
+fi
+
+checker=0
+while [[ $# -gt 0 ]]
+do
+key="$1"
+
+case $key in
+    -h|--help)
+    shift
+    usage
+    exit 0
+    ;;
+    -sb|--src-branch)
+    SOURCE_BRANCH="$2"
+    let "checker|=0x1"
+    shift
+    shift
+    ;;
+    -sc|--src-commit)
+    SOURCE_COMMIT_ID="$2"
+    let "checker|=0x2"
+    shift
+    shift
+    ;;
+    -tb|--target-branch)
+    TARGET_BRANCH="$2"
+    let "checker|=0x4"
+    shift
+    shift
+    ;;
+    -tc|--target-commit)
+    TARGET_COMMIT_ID="$2"
+    let "checker|=0x8"
+    shift
+    shift
+    ;;
+    *)
+    echo "Syntax Error: unknown option: $key"
+    echo ""
+    usage
+    exit 1
+esac
+
+done
+
+# If the repo workspace is shallow, unshallow it
+NB_COMMITS=`git log --oneline | wc -l`
+if [ $NB_COMMITS -eq 1 ]
+then
+    git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" > /dev/null 2>&1
+    echo "git remote update"
+    git remote update > /dev/null 2>&1
+    echo "git fetch --prune --unshallow"
+    git fetch --prune --unshallow > /dev/null 2>&1
+fi
+
+if [[ $TARGET_COMMIT_ID == "latest" ]]
+then
+    TARGET_COMMIT_ID=`git log -n1 --pretty=format:%H origin/$TARGET_BRANCH`
+fi
+
+echo "Source Branch is    : $SOURCE_BRANCH"
+echo "Source Commit ID is : $SOURCE_COMMIT_ID"
+echo "Target Branch is    : $TARGET_BRANCH"
+echo "Target Commit ID is : $TARGET_COMMIT_ID"
+
+if [ $checker -ne 15 ]
+then
+    echo ""
+    echo "Syntax Error: missing option"
+    echo ""
+    usage
+    exit 1
+fi
+
+git config user.email "jenkins@openairinterface.org"
+git config user.name "OAI Jenkins"
+
+git checkout -f $SOURCE_COMMIT_ID > checkout.txt 2>&1
+STATUS=`egrep -c "fatal: reference is not a tree" checkout.txt`
+rm -f checkout.txt
+if [ $STATUS -ne 0 ]
+then
+    echo "fatal: reference is not a tree --> $SOURCE_COMMIT_ID"
+    STATUS=-1
+    exit $STATUS
+fi
+
+git log -n1 --pretty=format:\"%s\" > .git/CI_COMMIT_MSG
+
+git merge --ff $TARGET_COMMIT_ID -m "Temporary merge for CI"
+
+STATUS=`git status | egrep -c "You have unmerged paths.|fix conflicts"`
+if [ $STATUS -ne 0 ]
+then
+    echo "There are merge conflicts.. Cannot perform further build tasks"
+    STATUS=-1
+fi
+exit $STATUS