#!/bin/groovy
/*
 * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The OpenAirInterface Software Alliance licenses this file to You under
 * the OAI Public License, Version 1.1  (the "License"); you may not use this file
 * except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.openairinterface.org/?page_id=698
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *-------------------------------------------------------------------------------
 * For more information about the OpenAirInterface (OAI) Software Alliance:
 *      contact@openairinterface.org
 */

//-------------------------------------------------------------------------------

// Location of the CN executor node
def cn_ci_host = params.Host_CN_CI_Server

// for lock
def cn_ci_resource = params.DockerContainers

// Location of the 2nd CN executor
def new_host_flag = false
def new_host = ""
def new_host_user = ""

// Tags/Branches to use
def smf_tag = "develop"
def smf_branch = "develop"

//-------------------------------------------------------------------------------
// Pipeline start
pipeline {
  agent {
    label cn_ci_host
  }
  options {
    disableConcurrentBuilds()
    timestamps()
    ansiColor('xterm')
    lock(cn_ci_resource)
    gitLabConnection('OAI GitLab')
    gitlabBuilds(builds: [
      "Build SMF Image",
      "Static Code Analysis",
      "Code Formatting Checker",
      "Build AMF-server",
      "Build UDM-server",
      "Build AMF-client",
      "Local SMF test",
      "Test with DsTester"
    ])

  }

  stages {
    stage ('Verify Parameters') {
      steps {
        script {
          echo '\u2705 \u001B[32mVerify Parameters\u001B[0m'

          JOB_TIMESTAMP = sh returnStdout: true, script: 'date --utc --rfc-3339=seconds | sed -e "s#+00:00##"'
          JOB_TIMESTAMP = JOB_TIMESTAMP.trim()

          if (params.Host_CN_CI_2nd_Server_Flag != null) {
            new_host_flag = params.Host_CN_CI_2nd_Server_Flag
            if (new_host_flag) {
              new_host = params.Host_CN_CI_2nd_Server
              new_host_user = params.Host_CN_CI_2nd_Server_Login
              echo "1st Node   is ${NODE_NAME}"
              echo "2nd Node   is ${new_host}"
            } else {
              echo "Node       is ${NODE_NAME}"
            }
          } else {
            echo "Node       is ${NODE_NAME}"
          }
          echo "Git URL    is ${GIT_URL}"
        }
      }
    }
    stage ('Prepare Source Code') {
      steps {
        script {
          sh "git clean -x -d -f > /dev/null 2>&1"
          if ("MERGE".equals(env.gitlabActionType)) {
            gitCommitAuthorEmailAddr = env.gitlabUserEmail
            echo "GitLab Usermail is ${gitCommitAuthorEmailAddr}"
            sh "./ci-scripts/doGitLabMerge.sh --src-branch ${env.gitlabSourceBranch} --src-commit ${env.gitlabMergeRequestLastCommit} --target-branch ${env.gitlabTargetBranch} --target-commit ${GIT_COMMIT}"
            smf_tag = "ci-tmp"
            smf_branch = env.gitlabSourceBranch
          } else {
            echo "Git Branch      is ${GIT_BRANCH}"
            echo "Git Commit      is ${GIT_COMMIT}"
            gitCommitAuthorEmailAddr = sh returnStdout: true, script: 'git log -n1 --pretty=format:%ae ${GIT_COMMIT}'
            gitCommitAuthorEmailAddr = gitCommitAuthorEmailAddr.trim()
            echo "GitLab Usermail is ${gitCommitAuthorEmailAddr}"
            sh "git log -n1 --pretty=format:\"%s\" > .git/CI_COMMIT_MSG"
          }
          sh "tar -cjhf /tmp/openair-smf.tar.bz2 ."
          sh "mv /tmp/openair-smf.tar.bz2 ."
          copyTo2ndServer('openair-smf.tar.bz2', new_host_flag, new_host_user, new_host)
          sh "mkdir -p archives"
          if (new_host_flag) {
            sh "mkdir -p archives/oai-smf-cfg"
          }
        }
      }
      post {
        failure {
          script {
            def message = "OAI " + JOB_NAME + " build (" + BUILD_ID + "): Merge Conflicts -- Cannot perform CI"
            addGitLabMRComment comment: message
            currentBuild.result = 'FAILURE'
          }
        }
      }
    }
    stage('Build Core Network Function') {
      parallel {
        stage ('Build SMF Image') {
          steps {
            script {
              gitlabCommitStatus(name: "Build SMF Image") {
                myShCmd('docker image prune --force', new_host_flag, new_host_user, new_host)
                if ("PUSH".equals(env.gitlabActionType)) {
                  // Currently this pipeline only runs for pushes to `develop` branch
                  // First clean image registry
                  try {
                    myShCmd('docker image rm oai-smf:develop', new_host_flag, new_host_user, new_host)
                  } catch (Exception e) {
                    echo "Maybe a previous build went wrong"
                  }
                }
                myShCmd('docker build --target oai-smf --tag oai-smf:' + smf_tag + ' --file docker/Dockerfile.ubuntu18.04 --build-arg NEEDED_GIT_PROXY="http://proxy.eurecom.fr:8080" . > archives/smf_docker_image_build.log 2>&1', new_host_flag, new_host_user, new_host)
                myShCmd('docker image ls >> archives/smf_docker_image_build.log', new_host_flag, new_host_user, new_host)
              }
            }
          }
          post {
            always {
              script {
                copyFrom2ndServer('archives/smf_docker_image_build.log', 'archives', new_host_flag, new_host_user, new_host)
              }
            }
            success {
              sh "echo 'OAI-SMF DOCKER IMAGE BUILD: OK' >> archives/smf_docker_image_build.log"
            }
            unsuccessful {
              sh "echo 'OAI-SMF DOCKER IMAGE BUILD: KO' >> archives/smf_docker_image_build.log"
            }
          }
        }
        // Running CPPCHECK in parallel to gain time
        stage ('Static Code Analysis') {
          steps {
            script {
              gitlabCommitStatus(name: "Static Code Analysis") {
                // Running on xenial to have 1.72 version of cppcheck
                myShCmd('docker run --name ci-cn-cppcheck -d ubuntu:xenial /bin/bash -c "sleep infinity"', new_host_flag, new_host_user, new_host)
                myShCmd('docker exec -it ci-cn-cppcheck /bin/bash -c "apt-get update && apt-get upgrade --yes" > archives/cppcheck_install.log', new_host_flag, new_host_user, new_host)
                myShCmd('docker exec -it ci-cn-cppcheck /bin/bash -c "apt-get install --yes cppcheck bzip2" >> archives/cppcheck_install.log', new_host_flag, new_host_user, new_host)

                myShCmd('docker cp ./openair-smf.tar.bz2 ci-cn-cppcheck:/home', new_host_flag, new_host_user, new_host)
                myShCmd('docker exec -it ci-cn-cppcheck /bin/bash -c "cd /home && tar -xjf openair-smf.tar.bz2"', new_host_flag, new_host_user, new_host)
                myShCmd('docker exec -it ci-cn-cppcheck /bin/bash -c "rm -f /home/openair-smf.tar.bz2"', new_host_flag, new_host_user, new_host)

                myShCmd('docker exec -it ci-cn-cppcheck /bin/bash -c "cd /home && cppcheck --enable=warning --force --xml --xml-version=2 --suppressions-list=ci-scripts/cppcheck_suppressions.list src 2> cppcheck.xml 1> cppcheck_build.log"', new_host_flag, new_host_user, new_host)
              }
            }
          }
          post {
            always {
              script {
                myShCmd('docker cp ci-cn-cppcheck:/home/cppcheck.xml archives', new_host_flag, new_host_user, new_host)
                myShCmd('docker cp ci-cn-cppcheck:/home/cppcheck_build.log archives', new_host_flag, new_host_user, new_host)
                copyFrom2ndServer('archives/cppcheck*.*', 'archives', new_host_flag, new_host_user, new_host)
                // no need to keep the cppcheck container
                myShCmd('docker rm -f ci-cn-cppcheck', new_host_flag, new_host_user, new_host)
              }
            }
            success {
              sh "echo 'CPPCHECK: OK' >> archives/cppcheck_install.log"
            }
            unsuccessful {
              sh "echo 'CPPCHECK: KO' >> archives/cppcheck_install.log"
            }
          }
        }
        // Running CLANG-FORMATTING check in parallel to gain time
        stage ('Code Formatting Checker') {
          steps {
            script {
              gitlabCommitStatus(name: "Code Formatting Checker") {
                myShCmd('docker run --name ci-cn-clang-formatter -d ubuntu:bionic /bin/bash -c "sleep infinity"', new_host_flag, new_host_user, new_host)
                myShCmd('docker exec -it ci-cn-clang-formatter /bin/bash -c "apt-get update && apt-get upgrade --yes" > archives/clang_format_install.log', new_host_flag, new_host_user, new_host)
                myShCmd('docker exec -it ci-cn-clang-formatter /bin/bash -c "apt-get install --yes git tree bzip2" >> archives/clang_format_install.log', new_host_flag, new_host_user, new_host)

                myShCmd('docker cp ./openair-smf.tar.bz2 ci-cn-clang-formatter:/home', new_host_flag, new_host_user, new_host)
                myShCmd('docker exec -it ci-cn-clang-formatter /bin/bash -c "cd /home && tar -xjf openair-smf.tar.bz2"', new_host_flag, new_host_user, new_host)
                myShCmd('docker exec -it ci-cn-clang-formatter /bin/bash -c "rm -f /home/openair-smf.tar.bz2"', new_host_flag, new_host_user, new_host)

                // We install a dedicated version (installed on our CI server).
                myShCmd('docker cp /opt/clang-format/9.0.0/bin/clang-format ci-cn-clang-formatter:/usr/local/bin', new_host_flag, new_host_user, new_host)
                if ("MERGE".equals(env.gitlabActionType)) {
                  myShCmd('docker exec -it ci-cn-clang-formatter /bin/bash -c "cd /home && ./ci-scripts/checkCodingFormattingRules.sh --src-branch ' + env.gitlabSourceBranch +' --target-branch ' + env.gitlabTargetBranch + '"', new_host_flag, new_host_user, new_host)
                } else {
                  myShCmd('docker exec -it ci-cn-clang-formatter /bin/bash -c "cd /home && ./ci-scripts/checkCodingFormattingRules.sh"', new_host_flag, new_host_user, new_host)
                }
              }
            }
          }
          post {
            always {
              script {
                myShCmd('docker cp ci-cn-clang-formatter:/home/src/oai_rules_result.txt src', new_host_flag, new_host_user, new_host)
                // May not have been generated
                try {
                  myShCmd('docker cp ci-cn-clang-formatter:/home/src/oai_rules_result_list.txt src', new_host_flag, new_host_user, new_host)
                } catch (Exception e) {
                  echo "Failed to copy src/oai_rules_result_list.txt! It may not have been generated. That's OK!"
                }
                copyFrom2ndServer('archives/clang_format*.*', 'archives', new_host_flag, new_host_user, new_host)
                copyFrom2ndServer('src/oai_rules*.*', 'src', new_host_flag, new_host_user, new_host)
                // no need to keep the clang-formatter container
                myShCmd('docker rm -f ci-cn-clang-formatter', new_host_flag, new_host_user, new_host)
              }
            }
          }
        }
      }
    }
    stage('Build Test Images') {
      stages {
        stage('Create Builder-Image') {
          steps {
            script {
              BUILDER_SHA_ONE = myShRetCmd('docker image ls | head -3 | grep none | sed -e "s#> *<#><#" | sed -e "s#<none><none> *##" | cut -d " " -f 1', new_host_flag, new_host_user, new_host)
              myShCmd('docker image tag ' + BUILDER_SHA_ONE + ' oai-smf-builder-installed:latest', new_host_flag, new_host_user, new_host)
            }
          }
        }
        stage('Build Test AMF-server') {
          steps {
            script {
              gitlabCommitStatus(name: "Build AMF-server") {
                myShCmd('docker build --target test-amf-server --tag test-amf-server:test-deploy --file docker/test-images/Dockerfile.amf-server.ubuntu18.04 . > archives/amf_server_docker_image_build.log 2>&1', new_host_flag, new_host_user, new_host)
                myShCmd('docker image ls >> archives/amf_server_docker_image_build.log', new_host_flag, new_host_user, new_host)
              }
            }
          }
          post {
            always {
              script {
                copyFrom2ndServer('archives/amf_server_docker_image_build.log', 'archives', new_host_flag, new_host_user, new_host)
              }
            }
            success {
              sh "echo 'TEST-AMF-SERVER DOCKER IMAGE BUILD: OK' >> archives/amf_server_docker_image_build.log"
            }
            unsuccessful {
              sh "echo 'TEST-AMF-SERVER DOCKER IMAGE BUILD: KO' >> archives/amf_server_docker_image_build.log"
            }
          }
        }
        stage('Build Test UDM-server') {
          steps {
            script {
              gitlabCommitStatus(name: "Build UDM-server") {
                myShCmd('docker build --target test-udm-server --tag test-udm-server:test-deploy --file docker/test-images/Dockerfile.udm-server.ubuntu18.04 . > archives/udm_server_docker_image_build.log 2>&1', new_host_flag, new_host_user, new_host)
                myShCmd('docker image ls >> archives/udm_server_docker_image_build.log', new_host_flag, new_host_user, new_host)
              }
            }
          }
          post {
            always {
              script {
                copyFrom2ndServer('archives/udm_server_docker_image_build.log', 'archives', new_host_flag, new_host_user, new_host)
              }
            }
            success {
              sh "echo 'TEST-UDM-SERVER DOCKER IMAGE BUILD: OK' >> archives/udm_server_docker_image_build.log"
            }
            unsuccessful {
              sh "echo 'TEST-UDM-SERVER DOCKER IMAGE BUILD: KO' >> archives/udm_server_docker_image_build.log"
            }
          }
        }
        stage('Build Test AMF-client') {
          steps {
            script {
              gitlabCommitStatus(name: "Build AMF-client") {
                myShCmd('docker build --target test-amf-client --tag test-amf-client:test-deploy --file docker/test-images/Dockerfile.amf-client.ubuntu18.04 . > archives/amf_client_docker_image_build.log 2>&1', new_host_flag, new_host_user, new_host)
                myShCmd('docker image ls >> archives/amf_client_docker_image_build.log', new_host_flag, new_host_user, new_host)
              }
            }
          }
          post {
            always {
              script {
                copyFrom2ndServer('archives/amf_client_docker_image_build.log', 'archives', new_host_flag, new_host_user, new_host)
              }
            }
            success {
              sh "echo 'TEST-AMF-CLIENT DOCKER IMAGE BUILD: OK' >> archives/amf_client_docker_image_build.log"
            }
            unsuccessful {
              sh "echo 'TEST-AMF-CLIENT DOCKER IMAGE BUILD: KO' >> archives/amf_client_docker_image_build.log"
            }
          }
        }
      }
    }
    stage('Deploy Simple Test') {
      stages {
        stage ('Create Networks') {
          steps {
            script {
              myShCmd('docker network create --attachable --subnet 192.168.28.0/24 --ip-range 192.168.28.0/24 ci-sbi', new_host_flag, new_host_user, new_host)
              myShCmd('docker network create --attachable --subnet 192.168.29.0/24 --ip-range 192.168.29.0/24 ci-n4', new_host_flag, new_host_user, new_host)
            }
          }
        }
        stage ('Deploy Containers') {
          steps {
            script {
              myShCmd('docker run --privileged --name ci-oai-smf -d oai-smf:' + smf_tag + ' /bin/bash -c "sleep infinity"', new_host_flag, new_host_user, new_host)
              // SBI --> eth1
              myShCmd('docker network connect ci-sbi ci-oai-smf', new_host_flag, new_host_user, new_host)
              // N4 --> eth2
              myShCmd('docker network connect ci-n4 ci-oai-smf', new_host_flag, new_host_user, new_host)
              // Test containers
              myShCmd('docker run --name ci-test-amf-server -d test-amf-server:test-deploy /bin/bash -c "sleep infinity"', new_host_flag, new_host_user, new_host)
              myShCmd('docker network connect ci-sbi ci-test-amf-server', new_host_flag, new_host_user, new_host)
              myShCmd('docker run --name ci-test-udm-server -d test-udm-server:test-deploy /bin/bash -c "sleep infinity"', new_host_flag, new_host_user, new_host)
              myShCmd('docker network connect ci-sbi ci-test-udm-server', new_host_flag, new_host_user, new_host)
              myShCmd('docker run --name ci-test-amf-client -d test-amf-client:test-deploy /bin/bash -c "sleep infinity"', new_host_flag, new_host_user, new_host)
              myShCmd('docker network connect ci-sbi ci-test-amf-client', new_host_flag, new_host_user, new_host)
              // SPGWU as UPF
              myShCmd('docker run --privileged --name ci-oai-spgwu -d oai-spgwu-tiny:develop /bin/bash -c "sleep infinity"', new_host_flag, new_host_user, new_host)
              // N4 --> eth1
              myShCmd('docker network connect ci-n4 ci-oai-spgwu', new_host_flag, new_host_user, new_host)

              sh "echo 'OAI-AMF-SERVER CONFIG: OK' >> archives/amf_server_config.log"
              sh "echo 'OAI-UDM-SERVER CONFIG: OK' >> archives/udm_server_config.log"
              sh "echo 'OAI-AMF-CLIENT CONFIG: OK' >> archives/amf_client_config.log"

              // Configure containers
              SMF_SBI_IP_ADDRESS=myShRetCmd('docker exec -it ci-oai-smf /bin/bash -c "ifconfig eth1 | egrep inet" | sed -e "s@^.*inet @@" -e "s@  netmask.*@@"', new_host_flag, new_host_user, new_host)
              SMF_N4_IP_ADDRESS=myShRetCmd('docker exec -it ci-oai-smf /bin/bash -c "ifconfig eth2 | egrep inet" | sed -e "s@^.*inet @@" -e "s@  netmask.*@@"', new_host_flag, new_host_user, new_host)
              UPF_N4_IP_ADDRESS=myShRetCmd('docker exec -it ci-oai-spgwu /bin/bash -c "ifconfig eth1 | egrep inet" | sed -e "s@^.*inet @@" -e "s@  netmask.*@@"', new_host_flag, new_host_user, new_host)
              AMF_IP_ADDRESS=myShRetCmd('docker exec -it ci-test-amf-server /bin/bash -c "ifconfig eth1 | egrep inet" | sed -e "s@^.*inet @@" -e "s@  netmask.*@@"', new_host_flag, new_host_user, new_host)
              UDM_IP_ADDRESS=myShRetCmd('docker exec -it ci-test-udm-server /bin/bash -c "ifconfig eth1 | egrep inet" | sed -e "s@^.*inet @@" -e "s@  netmask.*@@"', new_host_flag, new_host_user, new_host)

              sh "python3 ci-scripts/generateConfigFiles.py --kind=SMF --sbi=eth1 --n4=eth2 --amf_ip_addr=${AMF_IP_ADDRESS} --udm_ip_addr=${UDM_IP_ADDRESS} --upf_ip_addr=${UPF_N4_IP_ADDRESS} --from_docker_file"
              copyTo2ndServer('smf-cfg.sh', new_host_flag, new_host_user, new_host)
              myShCmd('docker cp ./smf-cfg.sh ci-oai-smf:/openair-smf', new_host_flag, new_host_user, new_host)
              myShCmd('docker exec -it ci-oai-smf /bin/bash -c "cd /openair-smf && chmod 777 smf-cfg.sh && ./smf-cfg.sh" >> archives/smf_config.log', new_host_flag, new_host_user, new_host)
              myShCmd('docker cp ci-oai-smf:/openair-smf/etc/. archives/oai-smf-cfg', new_host_flag, new_host_user, new_host)
              copyFrom2ndServer('archives/oai-smf-cfg/*.*', 'archives/oai-smf-cfg', new_host_flag, new_host_user, new_host)
              copyFrom2ndServer('archives/smf_config.log', 'archives', new_host_flag, new_host_user, new_host)

              sh "wget --quiet https://raw.githubusercontent.com/OPENAIRINTERFACE/openair-spgwu-tiny/develop/ci-scripts/generateConfigFiles.py -O ci-scripts/generateSpgwuConfigFiles.py"
              sh "python3 ci-scripts/generateSpgwuConfigFiles.py --kind=SPGW-U --sxc_ip_addr=${SMF_N4_IP_ADDRESS} --sxu=eth1 --s1u=eth0 --from_docker_file"
              copyTo2ndServer('spgwu-cfg.sh', new_host_flag, new_host_user, new_host)
              myShCmd('docker cp ./spgwu-cfg.sh ci-oai-spgwu:/openair-spgwu-tiny', new_host_flag, new_host_user, new_host)
              myShCmd('docker exec -it ci-oai-spgwu /bin/bash -c "cd /openair-spgwu-tiny && chmod 777 spgwu-cfg.sh && ./spgwu-cfg.sh" >> archives/spgwu_config.log', new_host_flag, new_host_user, new_host)
              copyFrom2ndServer('archives/spgwu_config.log', 'archives', new_host_flag, new_host_user, new_host)
            }
          }
          post {
            success {
              sh "echo 'OAI-SMF CONFIG: OK' >> archives/smf_config.log"
              sh "echo 'OAI-SPGW-U CONFIG: OK' >> archives/spgwu_config.log"
            }
            unsuccessful {
              sh "echo 'OAI-SMF CONFIG: KO' >> archives/smf_config.log"
              sh "echo 'OAI-SPGW-U CONFIG: KO' >> archives/spgwu_config.log"
            }
          }
        }
        stage ('Test') {
          steps {
            script {
              gitlabCommitStatus(name: "Local SMF test") {
                  SMF_SBI_IP_ADDRESS=myShRetCmd('docker exec -it ci-oai-smf /bin/bash -c "ifconfig eth1 | egrep inet" | sed -e "s@^.*inet @@" -e "s@  netmask.*@@"', new_host_flag, new_host_user, new_host)
                  AMF_IP_ADDRESS=myShRetCmd('docker exec -it ci-test-amf-server /bin/bash -c "ifconfig eth1 | egrep inet" | sed -e "s@^.*inet @@" -e "s@  netmask.*@@"', new_host_flag, new_host_user, new_host)
                  UDM_IP_ADDRESS=myShRetCmd('docker exec -it ci-test-udm-server /bin/bash -c "ifconfig eth1 | egrep inet" | sed -e "s@^.*inet @@" -e "s@  netmask.*@@"', new_host_flag, new_host_user, new_host)

                  myShCmd('docker exec -d ci-oai-smf /bin/bash -c "nohup ./bin/oai_smf -c ./etc/smf.conf -o > smf_check_run.log 2>&1"', new_host_flag, new_host_user, new_host)
                  sh "sleep 2"
                  myShCmd('docker exec -d  ci-oai-spgwu /bin/bash -c "nohup ./bin/oai_spgwu -c ./etc/spgw_u.conf -o > spgwu_check_run.log 2>&1"', new_host_flag, new_host_user, new_host)
                  sh "sleep 2"
                  myShCmd('docker exec -d ci-test-udm-server /bin/bash -c "nohup ./bin/udm-server -i ' + UDM_IP_ADDRESS + ' > udm_server_check_run.log 2>&1"', new_host_flag, new_host_user, new_host)
                  sh "sleep 2"
                  myShCmd('docker exec -d ci-test-amf-server /bin/bash -c "nohup ./bin/amf-server -i ' + AMF_IP_ADDRESS + ' > amf_server_check_run.log 2>&1"', new_host_flag, new_host_user, new_host)

                  sh "sleep 30"
                  myShCmd('docker exec -it ci-test-amf-client /bin/bash -c "./bin/amf-client -i ' + SMF_SBI_IP_ADDRESS + ' > amf_client_check_run.log"', new_host_flag, new_host_user, new_host)
                  sh "sleep 30"

                  myShCmd('docker exec -it ci-oai-smf /bin/bash -c "killall --signal SIGINT oai_smf"', new_host_flag, new_host_user, new_host)
                  myShCmd('docker exec -it ci-oai-spgwu /bin/bash -c "killall --signal SIGINT oai_spgwu"', new_host_flag, new_host_user, new_host)
                  myShCmd('docker exec -it ci-test-amf-server /bin/bash -c "killall amf-server"', new_host_flag, new_host_user, new_host)
                  myShCmd('docker exec -it ci-test-udm-server /bin/bash -c "killall udm-server"', new_host_flag, new_host_user, new_host)
                  sh "sleep 5"
                  try {
                    myShCmd('docker exec -it ci-oai-smf /bin/bash -c "killall --signal SIGKILL oai_smf"', new_host_flag, new_host_user, new_host)
                  } catch (Exception e) {
                    echo "No residual oai_smf processes"
                  }
                  try {
                    myShCmd('docker exec -it ci-oai-spgwu /bin/bash -c "killall --signal SIGKILL oai_spgwu"', new_host_flag, new_host_user, new_host)
                  } catch (Exception e) {
                    echo "No residual oai_spgwu processes"
                  }
                  sh "sleep 5"

                  // Retrieve all log files
                  myShCmd('docker cp ci-oai-smf:/openair-smf/smf_check_run.log archives', new_host_flag, new_host_user, new_host)
                  myShCmd('docker cp ci-oai-spgwu:/openair-spgwu-tiny/spgwu_check_run.log archives', new_host_flag, new_host_user, new_host)
                  myShCmd('docker cp ci-test-amf-server:/test-amf-server/amf_server_check_run.log archives', new_host_flag, new_host_user, new_host)
                  myShCmd('docker cp ci-test-udm-server:/test-udm-server/udm_server_check_run.log archives', new_host_flag, new_host_user, new_host)
                  myShCmd('docker cp ci-test-amf-client:/test-amf-client/amf_client_check_run.log archives', new_host_flag, new_host_user, new_host)
                  copyFrom2ndServer('archives/*_check_run.log', 'archives', new_host_flag, new_host_user, new_host)
                  sh "python3 ./ci-scripts/verifySanityCheckDeployment.py --job_name=${JOB_NAME} --job_id=${BUILD_ID}"
                  sh "python3 ./ci-scripts/verifySanityCheckDeployment.py --job_name=${JOB_NAME} --job_id=${BUILD_ID} --smf_pdu_check"
              }
            }
          }
          post {
            unsuccessful {
              script {
                // Retrieve all log files in case of failure
                try {
                  myShCmd('docker cp ci-oai-smf:/openair-smf/smf_check_run.log archives', new_host_flag, new_host_user, new_host)
                } catch (Exception e) {
                  echo "No smf run logs"
                }
                try {
                  myShCmd('docker cp ci-oai-spgwu:/openair-spgwu-tiny/spgwu_check_run.log archives', new_host_flag, new_host_user, new_host)
                } catch (Exception e) {
                  echo "No spgwu run logs"
                }
                try {
                  myShCmd('docker cp ci-test-amf-server:/test-amf-server/amf_server_check_run.log archives', new_host_flag, new_host_user, new_host)
                } catch (Exception e) {
                  echo "No amf run logs"
                }
                try {
                  myShCmd('docker cp ci-test-udm-server:/test-udm-server/udm_server_check_run.log archives', new_host_flag, new_host_user, new_host)
                } catch (Exception e) {
                  echo "No udm run logs"
                }
                try {
                  myShCmd('docker cp ci-test-amf-client:/test-amf-client/amf_client_check_run.log archives', new_host_flag, new_host_user, new_host)
                } catch (Exception e) {
                  echo "No amf-client run logs"
                }
                try {
                  copyFrom2ndServer('archives/*_check_run.log', 'archives', new_host_flag, new_host_user, new_host)
                } catch (Exception e) {
                  echo "No run logs at all?"
                }
              }
            }
          }
        }
        stage ('Undeploy Containers') {
          steps {
            script {
              myShCmd('docker rm -f ci-oai-smf ci-test-amf-server ci-test-udm-server ci-test-amf-client ci-oai-spgwu', new_host_flag, new_host_user, new_host)
            }
          }
        }
        stage ('Delete Networks') {
          steps {
            script {
              myShCmd('docker network rm ci-sbi ci-n4', new_host_flag, new_host_user, new_host)
            }
          }
        }
      }
    }
    stage ('Testing in CN-5G-FED environment') {
      steps {
        script {
          gitlabCommitStatus(name: "Test with DsTester") {
            localStatus = build job: 'CI-OpenAir-CN5G-FED',
              parameters: [
                string(name: 'SMF_TAG', value: String.valueOf(smf_tag)),
                string(name: 'SMF_BRANCH', value: String.valueOf(smf_branch))
              ], propagate: false
            localResult = localStatus.getResult()

            if (localStatus.resultIsBetterOrEqualTo('SUCCESS')) {
              echo "Federation Test Job is OK"
            } else {
              echo "Federation Test Job is KO"
              sh "ci-scripts/fail.sh"
            }
          }
        }
      }
      post {
        always {
          script {
            copyArtifacts(projectName: 'CI-OpenAir-CN5G-FED',
                          filter: '*_results_oai_cn5g.html',
                          selector: lastCompleted())
          }
        }
      }
    }
  }
  post {
    success {
      script {
        if ("MERGE".equals(env.gitlabActionType)) {
          def message = "OAI " + JOB_NAME + " build (" + BUILD_ID + "): passed (" + BUILD_URL + ")"
          echo "This is a MERGE event"
          addGitLabMRComment comment: message
        }
      }
    }
    unsuccessful {
      script {
        if ("MERGE".equals(env.gitlabActionType)) {
          def message = "OAI " + JOB_NAME + " build (" + BUILD_ID + "): failed (" + BUILD_URL + ")"
          echo "This is a MERGE event"
          addGitLabMRComment comment: message
        }
      }
    }
    cleanup {
      script {
        // Deleting containers in case of errors
        try {
          myShCmd('docker rm -f ci-oai-smf ci-test-amf-server ci-test-udm-server ci-test-amf-client ci-oai-spgwu', new_host_flag, new_host_user, new_host)
        } catch (Exception e) {
          echo "We failed to remove any existing containers"
        }
        // Deleting networks in case of errors
        try {
          myShCmd('docker network rm ci-sbi ci-n4', new_host_flag, new_host_user, new_host)
        } catch (Exception e) {
          echo "We failed to remove any existing networks"
        }
        // Removing temporary / intermediate images
        try {
          if ("MERGE".equals(env.gitlabActionType)) {
            myShCmd('docker image rm --force oai-smf:ci-tmp', new_host_flag, new_host_user, new_host)
          }
        } catch (Exception e) {
          echo "We failed to delete the OAI-SMF temp image"
        }
        try {
          myShCmd('docker image rm --force oai-smf-builder-installed:latest test-amf-server:test-deploy test-udm-server:test-deploy test-amf-client:test-deploy', new_host_flag, new_host_user, new_host)
        } catch (Exception e) {
          echo "We failed to delete the test temp images"
        }
        try {
          myShCmd('docker image prune --force', new_host_flag, new_host_user, new_host)
        } catch (Exception e) {
          echo "We failed to prune all unneeded intermediate images"
        }

        // Zipping all archived log files
        sh "zip -r -qq docker_logs.zip archives"
        if (fileExists('docker_logs.zip')) {
          archiveArtifacts artifacts: 'docker_logs.zip'
        }

        // Generating the HTML report
        if ("MERGE".equals(env.gitlabActionType)) {
          sh "python3 ci-scripts/generateHtmlReport.py --job_name=${JOB_NAME} --job_id=${BUILD_ID} --job_url=${BUILD_URL} --git_url=${GIT_URL} --git_src_branch=${env.gitlabSourceBranch} --git_src_commit=${env.gitlabMergeRequestLastCommit} --git_pull_request=True --git_target_branch=${env.gitlabTargetBranch} --git_target_commit=${GIT_COMMIT}"
        } else {
          sh "python3 ci-scripts/generateHtmlReport.py --job_name=${JOB_NAME} --job_id=${BUILD_ID} --job_url=${BUILD_URL} --git_url=${GIT_URL} --git_src_branch=${GIT_BRANCH} --git_src_commit=${GIT_COMMIT}"
        }
        sh "sed -i -e 's#TEMPLATE_TIME#${JOB_TIMESTAMP}#' test_results_oai_smf.html"
        if (fileExists('test_results_oai_smf.html')) {
          archiveArtifacts artifacts: 'test_results_oai_smf.html'
        }
        if (fileExists('deploy_results_oai_cn5g.html')) {
          sh "sed -i -e 's#TEMPLATE_TIME#${JOB_TIMESTAMP}#' *_results_oai_cn5g.html"
          archiveArtifacts artifacts: '*_results_oai_cn5g.html'
        }

        // Sending email to commiter
        emailext attachmentsPattern: '*results*.html',
          body: '''Hi,

Here are attached HTML report files for $PROJECT_NAME - Build # $BUILD_NUMBER - $BUILD_STATUS!

Regards,
OAI CI Team''',
          replyTo: 'no-reply@openairinterface.org',
          subject: '$PROJECT_NAME - Build # $BUILD_NUMBER - $BUILD_STATUS!',
          to: gitCommitAuthorEmailAddr
      }
    }
  }
}

def copyTo2ndServer(filename, flag, user, host) {
  if (flag) {
    if ("openair-smf.tar.bz2".equals(filename)) {
      sh "ssh ${user}@${host} 'rm -rf /tmp/CI-CN-SMF'"
      sh "ssh ${user}@${host} 'mkdir -p /tmp/CI-CN-SMF'"
    }
    sh "scp ${filename} ${user}@${host}:/tmp/CI-CN-SMF"
    if ("openair-smf.tar.bz2".equals(filename)) {
      sh "ssh ${user}@${host} 'cd /tmp/CI-CN-SMF && tar -xjf ${filename}'"
      sh "ssh ${user}@${host} 'mkdir -p /tmp/CI-CN-SMF/archives'"
      sh "ssh ${user}@${host} 'mkdir -p /tmp/CI-CN-SMF/archives/oai-smf-cfg'"
    }
  }
}

def copyFrom2ndServer(filename, target, flag, user, host) {
  if (flag) {
    sh "scp ${user}@${host}:/tmp/CI-CN-SMF/${filename} ${target}"
  }
}

def myShCmd(cmd, flag, user, host) {
  if (flag) {
    sh "ssh -t -t ${user}@${host} 'cd /tmp/CI-CN-SMF && ${cmd}'"
  } else {
    sh "${cmd}"
  }
}

def myShRetCmd(cmd, flag, user, host) {
  if (flag) {
    ret = sh returnStdout: true, script: "ssh -t -t ${user}@${host} 'cd /tmp/CI-CN-SMF && ${cmd}'"
  } else {
    ret = sh returnStdout: true, script: "${cmd}"
  }
  ret = ret.trim()
  return ret
}
