Merge branch 'ci-adding-base-image-builds' into 'develop'

feat(ci): using base images for merge requests builds in order to speed up build-time

See merge request oai/cn5g/oai-cn5g-upf-vpp!20

ci-scripts/Jenkinsfile-GitLab-Docker

@@ -43,6 +43,11 @@ def rem_rhel8_resource = params.PodmanContainers
 def upf_tag = "develop"
 def upf_branch = "develop"
 
+// We are using a base image to speed up CI build.
+// This base image is potentially subject to changes over time.
+def VPP_UPF_U18_BASE_IMAGE_TAG = params.UpfU18BaseImageTag
+def VPP_UPF_RHEL7_BASE_IMAGE_TAG = params.UpfRhel7BaseImageTag
+
 // Merge Request Link
 def gitlabMergeRequestLink
 
@@ -61,7 +66,6 @@ pipeline {
     gitlabBuilds(builds: [
       "Build RHEL7 UPF-VPP Image",
       "Build U18 UPF-VPP Image"
-      //"Test with DsTester"
     ])
 
   }
@@ -97,6 +101,25 @@ pipeline {
             }
           }
           echo "Git URL    is ${GIT_URL}"
+
+          if ("MERGE".equals(env.gitlabActionType)) {
+            try {
+              myShCmd('docker image inspect --format=\'Size = {{.Size}} bytes\' vpp-upf-base:' + VPP_UPF_U18_BASE_IMAGE_TAG, rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
+            } catch (Exception e) {
+              currentBuild.result = 'FAILURE'
+              echo '\u26D4 \u001B[31mUbuntu18 Base Image does not exist\u001B[0m'
+              error "Stopping pipeline!"
+            }
+            if (rem_rhel_host_flag) {
+              try {
+                myShCmd('sudo podman image inspect --format=\'Size = {{.Size}} bytes\' vpp-upf-base:' + VPP_UPF_RHEL7_BASE_IMAGE_TAG, rem_rhel_host_flag, rem_rhel_host_user, rem_rhel_host)
+              } catch (Exception e) {
+                currentBuild.result = 'FAILURE'
+                echo '\u26D4 \u001B[31mRHEL7 Base Image does not exist\u001B[0m'
+                error "Stopping pipeline!"
+              }
+            }
+          }
           if (params.DockerHubCredentials == null) {
             echo '\u26D4 \u001B[31mNo Credentials to push to DockerHub!\u001B[0m'
             error "Stopping pipeline!"
@@ -161,11 +184,32 @@ pipeline {
             script {
               gitlabCommitStatus(name: "Build U18 UPF-VPP Image") {
                 myShCmd('docker image prune --force', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
+                myShCmd('mkdir -p archives', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
+                if ("PUSH".equals(env.gitlabActionType)) {
                   // Currently this pipeline only runs for pushes to `develop` branch
                   // First clean image registry
-                myShCmd('docker image rm oai-upf-vpp:' + upf_tag + ' || true', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
-                myShCmd('mkdir -p archives', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
+                  try {
+                    myShCmd('docker image rm oai-upf-vpp:' + upf_tag, rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
+                  } catch (Exception e) {
+                    echo "Maybe a previous build went wrong"
+                  }
+                  // In case of push to `develop` branch we build from scratch
                   myShCmd('docker build --no-cache --target oai-upf-vpp --tag oai-upf-vpp:' + upf_tag + ' --file docker/Dockerfile.upf-vpp.ubuntu --build-arg NEEDED_GIT_PROXY="http://proxy.eurecom.fr:8080" . > archives/upf_docker_image_build.log 2>&1', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
+                }
+                if ("MERGE".equals(env.gitlabActionType)) {
+                  try {
+                    // Checking if the CI Base image is still there.
+                    // If the inspect command fails, it's not there. If it passes, let remove tag.
+                    myShCmd('docker image inspect vpp-upf-base:latest > /dev/null 2>&1', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
+                    myShCmd('docker rmi vpp-upf-base:latest', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
+                    myShCmd('docker rmi oai-upf-vpp:' + upf_tag, rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
+                  } catch (Exception e) {
+                    echo 'No need to remove the CI base image'
+                  }
+                  // "latest" is the tag used in the docker file.
+                  myShCmd('docker image tag vpp-upf-base:' + VPP_UPF_U18_BASE_IMAGE_TAG + ' vpp-upf-base:latest', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
+                  myShCmd('docker build --no-cache --target oai-upf-vpp --tag oai-upf-vpp:' + upf_tag + ' --file ci-scripts/docker/Dockerfile.upf-vpp.ubuntu . > archives/upf_docker_image_build.log 2>&1', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
+                }
                 myShCmd('docker image ls >> archives/upf_docker_image_build.log', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
                 // We will keep also the TMP image in U18 case.
                 if ("MERGE".equals(env.gitlabActionType)) {
@@ -194,14 +238,33 @@ pipeline {
             lock (rem_rhel8_resource) {
               script {
                 gitlabCommitStatus(name: "Build RHEL7 UPF-VPP Image") {
+                  // First clean image registry
+                  myShCmd('mkdir -p archives', rem_rhel_host_flag, rem_rhel_host_user, rem_rhel_host)
                   myShCmd('sudo podman image rm oai-upf-vpp:' + upf_tag + ' || true', rem_rhel_host_flag, rem_rhel_host_user, rem_rhel_host)
                   myShCmd('sudo podman image prune --force', rem_rhel_host_flag, rem_rhel_host_user, rem_rhel_host)
+                  if ("PUSH".equals(env.gitlabActionType)) {
+                    // In case of push to `develop` branch we build from scratch
                     // Copy the RHEL Host certificates for building
                     myShCmd('mkdir -p ./etc-pki-entitlement ./rhsm-conf ./rhsm-ca', rem_rhel_host_flag, rem_rhel_host_user, rem_rhel_host)
                     myShCmd('cp /etc/pki/entitlement/*pem ./etc-pki-entitlement', rem_rhel_host_flag, rem_rhel_host_user, rem_rhel_host)
                     myShCmd('sudo cp /etc/rhsm/rhsm.conf ./rhsm-conf', rem_rhel_host_flag, rem_rhel_host_user, rem_rhel_host)
                     myShCmd('sudo cp /etc/rhsm/ca/*pem ./rhsm-ca', rem_rhel_host_flag, rem_rhel_host_user, rem_rhel_host)
                     myShCmd('sudo podman build --no-cache --target oai-upf-vpp --tag oai-upf-vpp:' + upf_tag + ' --file docker/Dockerfile.upf-vpp.rhel7 --build-arg NEEDED_GIT_PROXY="http://proxy.eurecom.fr:8080" . > archives/upf_podman_image_build.log 2>&1', rem_rhel_host_flag, rem_rhel_host_user, rem_rhel_host)
+                    myShCmd('sudo rm -Rf ./etc-pki-entitlement ./rhsm-conf ./rhsm-ca', rem_rhel_host_flag, rem_rhel_host_user, rem_rhel_host)
+                  }
+                  if ("MERGE".equals(env.gitlabActionType)) {
+                    try {
+                      // Checking if the CI Base image is still there.
+                      // If the inspect command fails, it's not there. If it passes, let remove tag.
+                      myShCmd('sudo podman image inspect vpp-upf-base:latest > /dev/null 2>&1', rem_rhel_host_flag, rem_rhel_host_user, rem_rhel_host)
+                      myShCmd('sudo podman rmi vpp-upf-base:latest', rem_rhel_host_flag, rem_rhel_host_user, rem_rhel_host)
+                    } catch (Exception e) {
+                      echo 'No need to remove the CI base image'
+                    }
+                    // "latest" is the tag used in the docker file.
+                    myShCmd('sudo podman image tag vpp-upf-base:' + VPP_UPF_RHEL7_BASE_IMAGE_TAG + ' vpp-upf-base:latest', rem_rhel_host_flag, rem_rhel_host_user, rem_rhel_host)
+                    myShCmd('sudo podman build --no-cache --target oai-upf-vpp --tag oai-upf-vpp:' + upf_tag + ' --file ci-scripts/docker/Dockerfile.upf-vpp.rhel7 . > archives/upf_podman_image_build.log 2>&1', rem_rhel_host_flag, rem_rhel_host_user, rem_rhel_host)
+                  }
                   myShCmd('sudo podman image ls >> archives/upf_podman_image_build.log', rem_rhel_host_flag, rem_rhel_host_user, rem_rhel_host)
                   if ("MERGE".equals(env.gitlabActionType)) {
                     myShCmd('sudo podman image tag oai-upf-vpp:' + upf_tag + ' oai-upf-vpp:' + rhel_upf_tag, rem_rhel_host_flag, rem_rhel_host_user, rem_rhel_host)
@@ -333,6 +396,11 @@ pipeline {
     }
     cleanup {
       script {
+        if ("MERGE".equals(env.gitlabActionType)) {
+          // Remove CI Base image tag
+          myShCmd('docker rmi vpp-upf-base:latest || true', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
+          myShCmd('sudo podman rmi vpp-upf-base:latest || true', rem_rhel_host_flag, rem_rhel_host_user, rem_rhel_host)
+        }
         // Removing temporary / intermediate images
         try {
           if ("MERGE".equals(env.gitlabActionType)) {

ci-scripts/docker/Dockerfile.upf-vpp.rhel7

+#/*
+# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
+# * contributor license agreements.  See the NOTICE file distributed with
+# * this work for additional information regarding copyright ownership.
+# * The OpenAirInterface Software Alliance licenses this file to You under
+# * the OAI Public License, Version 1.1  (the "License"); you may not use this file
+# * except in compliance with the License.
+# * You may obtain a copy of the License at
+# *
+# *      http://www.openairinterface.org/?page_id=698
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# *-------------------------------------------------------------------------------
+# * For more information about the OpenAirInterface (OAI) Software Alliance:
+# *      contact@openairinterface.org
+# */
+#---------------------------------------------------------------------
+#
+# Dockerfile for the VPP-UPG service
+#   Valid for RHEL 8.2
+#
+#---------------------------------------------------------------------
+
+#---------------------------------------------------------------------
+# BUILDER IMAGE
+#---------------------------------------------------------------------
+FROM vpp-upf-base:latest AS vpp-upf-builder
+
+WORKDIR /vpp-upf
+COPY . .
+
+# Applying vpp patches
+RUN git clone -b stable/2101 https://github.com/fdio/vpp.git && \
+    git clone -b stable/1.2 https://github.com/travelping/upg-vpp.git && \
+    cd upg-vpp && \
+    cd .. && \
+    mv upg-vpp/upf/ vpp/src/plugins/ && \
+    mv upg-vpp/vpp-patches/* scripts/patches/ && \
+    mv upg-vpp/vpp.spec / && \
+    ./scripts/apply_vpp_patches.sh
+
+RUN make build-release -C vpp/
+
+#---------------------------------------------------------------------
+# TARGET IMAGE
+#---------------------------------------------------------------------
+FROM registry.access.redhat.com/ubi7/ubi:latest as oai-upf-vpp
+ENV TZ=Europe/Paris
+
+# We install some debug tools for the moment in addition of mandatory libraries
+RUN yum repolist --disablerepo=* && \
+    yum update -y && \
+    yum install -y \
+      net-tools \
+      iputils \
+      tshark \
+      tzdata\
+      iproute \
+      numactl-devel \
+      wget \
+      https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm && \
+    wget http://repo.openfusion.net/centos7-x86_64/hyperscan-devel-5.3.0-1.of.el7.x86_64.rpm \
+         http://repo.openfusion.net/centos7-x86_64/hyperscan-5.3.0-1.of.el7.x86_64.rpm && \
+    rpm -i *.rpm && \
+    rm *.rpm && \
+    yum remove -y wget && \
+    rm -rf /var/lib/apt/lists/*
+
+WORKDIR /openair-upf/bin
+COPY --from=vpp-upf-builder \
+       /vpp-upf/scripts/entrypoint.sh \
+       /vpp-upf/vpp/build-root/install-vpp-native/vpp/bin/vpp \
+       /vpp-upf/vpp/build-root/install-vpp-native/vpp/bin/vppctl \
+       /vpp-upf/src/nrf_client.sh ./
+
+COPY --from=vpp-upf-builder \
+      /vpp-upf/scripts/upf_conf/init.conf \
+      /vpp-upf/scripts/upf_conf/startup_debug.conf \
+      /vpp-upf/scripts/upf_conf/upf_profile.json \
+      /openair-upf/etc/
+
+COPY --from=vpp-upf-builder \
+      /vpp-upf/vpp/build-root/install-vpp-native/vpp/lib/vpp_plugins/upf_plugin.so \
+      /vpp-upf/vpp/build-root/install-vpp-native/vpp/lib/vpp_plugins/dpdk_plugin.so \
+      /usr/lib64/vpp_plugins/
+
+COPY --from=vpp-upf-builder \
+       /vpp-upf/vpp/build-root/install-vpp-native/vpp/lib/libvppinfra.so.21.01.1 \
+       /vpp-upf/vpp/build-root/install-vpp-native/vpp/lib/libvnet.so.21.01.1 \
+       /vpp-upf/vpp/build-root/install-vpp-native/vpp/lib/libvlibmemory.so.21.01.1 \
+       /vpp-upf/vpp/build-root/install-vpp-native/vpp/lib/libvlib.so.21.01.1 \
+       /vpp-upf/vpp/build-root/install-vpp-native/vpp/lib/libsvm.so.21.01.1 \
+#       /vpp-upf/vpp/build-root/install-vpp-native/vpp/lib/libnat.so.21.01.1 \
+       /usr/lib64/
+
+RUN ldconfig
+
+RUN groupadd vpp
+
+WORKDIR /openair-upf
+COPY --from=vpp-upf-builder /vpp-upf/scripts/run.sh  .
+
+#expose
+EXPOSE 8085/udp 2152/udp
+
+ENTRYPOINT ["/openair-upf/bin/entrypoint.sh"]
+CMD ["/openair-upf/run.sh"]

ci-scripts/docker/Dockerfile.upf-vpp.ubuntu

+#/*
+# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
+# * contributor license agreements.  See the NOTICE file distributed with
+# * this work for additional information regarding copyright ownership.
+# * The OpenAirInterface Software Alliance licenses this file to You under
+# * the OAI Public License, Version 1.1  (the "License"); you may not use this file
+# * except in compliance with the License.
+# * You may obtain a copy of the License at
+# *
+# *      http://www.openairinterface.org/?page_id=698
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# *-------------------------------------------------------------------------------
+# * For more information about the OpenAirInterface (OAI) Software Alliance:
+# *      contact@openairinterface.org
+# */
+#---------------------------------------------------------------------
+#
+# Dockerfile for the VPP-UPF service
+#   Valid for Ubuntu-18.04 (bionic) and Ubuntu-20.04 (focal)
+#
+#---------------------------------------------------------------------
+#---------------------------------------------------------------------
+# BUILDER IMAGE
+#---------------------------------------------------------------------
+
+ARG BASE_IMAGE=ubuntu:18.04
+FROM vpp-upf-base:latest AS vpp-upf-builder
+
+WORKDIR /vpp-upf
+COPY . .
+
+# Applying vpp patches
+RUN git clone -b stable/2101 https://github.com/fdio/vpp.git && \
+    git clone -b stable/1.2 https://github.com/travelping/upg-vpp.git && \
+    cd upg-vpp && \
+    cd .. && \
+    mv upg-vpp/upf/ vpp/src/plugins/ && \
+    mv upg-vpp/vpp-patches/* scripts/patches/ && \
+    mv upg-vpp/vpp.spec / && \
+    ./scripts/apply_vpp_patches.sh
+
+# To Do - remove unnecessary pulgin build to optimise build time e.g. ipsec, dpdk etc.
+
+# Build vpp with Travelping upf plugin
+RUN make build-release -C vpp
+
+#---------------------------------------------------------------------
+# TARGET IMAGE
+#---------------------------------------------------------------------
+FROM $BASE_IMAGE as oai-upf-vpp
+ENV DEBIAN_FRONTEND=noninteractive
+ENV TZ=Europe/Paris
+
+RUN apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get upgrade --yes && \
+    DEBIAN_FRONTEND=noninteractive apt-get install --yes \
+        net-tools \
+        tshark \
+        tzdata\
+        libhyperscan-dev \
+        iproute2 \
+        iputils-ping \
+        curl \
+        libnuma1 \
+  && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /openair-upf/bin
+COPY --from=vpp-upf-builder \
+       /vpp-upf/scripts/entrypoint.sh \
+       /vpp-upf/vpp/build-root/install-vpp-native/vpp/bin/vpp \
+       /vpp-upf/vpp/build-root/install-vpp-native/vpp/bin/vppctl \
+       /vpp-upf/src/nrf_client.sh \
+       ./
+
+COPY --from=vpp-upf-builder \
+       /vpp-upf/scripts/upf_conf/init.conf \
+       /vpp-upf/scripts/upf_conf/startup_debug.conf \
+       /vpp-upf/scripts/upf_conf/upf_profile.json \
+       /openair-upf/etc/
+
+COPY --from=vpp-upf-builder \
+      /vpp-upf/vpp/build-root/install-vpp-native/vpp/lib/vpp_plugins/upf_plugin.so \
+      /vpp-upf/vpp/build-root/install-vpp-native/vpp/lib/vpp_plugins/dpdk_plugin.so \
+      /usr/lib/x86_64-linux-gnu/vpp_plugins/
+
+COPY --from=vpp-upf-builder \
+     /vpp-upf/vpp/build-root/install-vpp-native/vpp/lib/libvppinfra.so.21.01.1 \
+     /vpp-upf/vpp/build-root/install-vpp-native/vpp/lib/libvnet.so.21.01.1 \
+     /vpp-upf/vpp/build-root/install-vpp-native/vpp/lib/libvlibmemory.so.21.01.1 \
+     /vpp-upf/vpp/build-root/install-vpp-native/vpp/lib/libvlib.so.21.01.1 \
+     /vpp-upf/vpp/build-root/install-vpp-native/vpp/lib/libsvm.so.21.01.1 \
+#     /vpp-upf/vpp/build-root/install-vpp-native/vpp/lib/libnat.so.21.01.1 \
+     /usr/lib/x86_64-linux-gnu/
+
+RUN ldconfig
+
+RUN groupadd vpp
+
+WORKDIR /openair-upf
+COPY --from=vpp-upf-builder /vpp-upf/scripts/run.sh  .
+
+#expose
+EXPOSE 8085/udp 2152/udp
+
+ENTRYPOINT ["/openair-upf/bin/entrypoint.sh"]
+CMD ["/openair-upf/run.sh"]