Skip to content

O
OpenXG-AMF
Commit 5f7eef9c authored by Mohammed Ismail's avatar Mohammed Ismail
Browse files
Options

[CI] initial RHEL build framework 

Signed-off-by: default avatarMohammed Ismail <mohammed.ismail@openairinterface.org>
parent 6b9755dc
Hide whitespace changes
Inline Side-by-side
Showing with 133 additions and 53 deletions
build/scripts/build_helper
View file @ 5f7eef9c
......@@ -39,7 +39,7 @@ case "$OS_DISTRO" in
ubuntu) OS_BASEDISTRO="debian"; INSTALLER="apt-get"; CMAKE="cmake" ;;
esac
IS_CONTAINER=`egrep -c "docker|kubepods|podman|runc-buildah|libpod" /proc/self/cgroup`
IS_CONTAINER=`egrep -c "docker|kubepods|podman|buildah|libpod" /proc/self/cgroup || true`
if [ $IS_CONTAINER -eq 0 ]
then
......
ci-scripts/Jenkinsfile-GitLab-Docker
View file @ 5f7eef9c
......@@ -28,14 +28,20 @@ def cn_ci_host = params.Host_CN_CI_Server
// for lock
def cn_ci_resource = params.DockerContainers
// Location of the 2nd CN executor
def new_host_flag = false
def new_host = ""
def new_host_user = ""
// Location of the Remote Ubuntu18 CN executor
def rem_u18_host_flag = false
def rem_u18_host = ""
def rem_u18_host_user = ""
// Location of the Remote RHEL CN executor
def rem_rhel_host_flag = false
def rem_rhel_host = ""
def rem_rhel_host_user = ""
def rem_rhel8_resource = params.PodmanContainers
// Tags/Branches to use
def amf_tag = "develop"
def amf_branch = "develop"
def amf_tag = parms.amfTag
def amf_branch = params.amfBranch
//-------------------------------------------------------------------------------
// Pipeline start
......@@ -68,18 +74,26 @@ pipeline {
JOB_TIMESTAMP = JOB_TIMESTAMP.trim()
if (params.Host_CN_CI_2nd_Server_Flag != null) {
new_host_flag = params.Host_CN_CI_2nd_Server_Flag
if (new_host_flag) {
new_host = params.Host_CN_CI_2nd_Server
new_host_user = params.Host_CN_CI_2nd_Server_Login
rem_u18_host_flag = params.Host_CN_CI_2nd_Server_Flag
if (rem_u18_host_flag) {
rem_u18_host = params.Host_CN_CI_2nd_Server
rem_u18_host_user = params.Host_CN_CI_2nd_Server_Login
echo "1st Node is ${NODE_NAME}"
echo "2nd Node is ${new_host}"
echo "2nd Node is ${rem_u18_host}"
} else {
echo "Node is ${NODE_NAME}"
echo "U18 Node is ${NODE_NAME}"
}
} else {
echo "Node is ${NODE_NAME}"
}
if (params.Remote_RHEL_Server_Flag != null) {
rem_rhel_host_flag = params.Remote_RHEL_Server_Flag
if (rem_rhel_host_flag) {
rem_rhel_host = params.Remote_RHEL_Server_Name
rem_rhel_host_user = params.Remote_RHEL_Server_Login
echo "RHEL Node is ${rem_rhel_host}"
}
}
echo "Git URL is ${GIT_URL}"
}
}
......@@ -89,12 +103,22 @@ pipeline {
script {
sh "git clean -x -d -f > /dev/null 2>&1"
if ("MERGE".equals(env.gitlabActionType)) {
echo "========= THIS IS A MERGE REQUEST =========="
echo "MR ID is ${env.gitlabMergeRequestId}"
echo "MR TITLE is ${env.gitlabMergeRequestTitle}"
gitCommitAuthorEmailAddr = env.gitlabUserEmail
echo "GitLab Usermail is ${gitCommitAuthorEmailAddr}"
sh "git fetch --prune --unshallow"
shortenShaOne = sh returnStdout: true, script: 'git log -1 --pretty=format:"%h" ' + env.gitlabMergeRequestLastCommit
shortenShaOne = shortenShaOne.trim()
sh "./ci-scripts/doGitLabMerge.sh --src-branch ${env.gitlabSourceBranch} --src-commit ${env.gitlabMergeRequestLastCommit} --target-branch ${env.gitlabTargetBranch} --target-commit ${GIT_COMMIT}"
sh "sleep 10"
amf_tag = "ci-tmp"
rhel_amf_tag = 'ci-temp-pr-' + env.gitlabMergeRequestId + '-' + shortenShaOne
amf_branch = env.gitlabSourceBranch
echo "MR TAG is ${rhel_amf_tag}"
} else {
echo "======== THIS IS A PUSH REQUEST ========"
echo "Git Branch is ${GIT_BRANCH}"
echo "Git Commit is ${GIT_COMMIT}"
gitCommitAuthorEmailAddr = sh returnStdout: true, script: 'git log -n1 --pretty=format:%ae ${GIT_COMMIT}'
......@@ -104,11 +128,9 @@ pipeline {
}
sh "tar -cjhf /tmp/openair-amf.tar.bz2 ."
sh "mv /tmp/openair-amf.tar.bz2 ."
copyTo2ndServer('openair-amf.tar.bz2', new_host_flag, new_host_user, new_host)
sh "mkdir -p archives"
if (new_host_flag) {
sh "mkdir -p archives/oai-amf-cfg"
}
copyTo2ndServer('openair-amf.tar.bz2', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
copyTo2ndServer('openair-amf.tar.bz2', rem_rhel_host_flag, rem_rhel_host_user, rem_rhel_host)
sh "mkdir -p archives/oai-amf-cfg"
}
}
post {
......@@ -123,29 +145,29 @@ pipeline {
}
stage('Build Core Network Function') {
parallel {
stage ('Build AMF Image') {
stage ('Build U18 AMF Image') {
steps {
script {
gitlabCommitStatus(name: "Build AMF Image") {
myShCmd('docker image prune --force', new_host_flag, new_host_user, new_host)
gitlabCommitStatus(name: "Build U18 AMF Image") {
myShCmd('docker image prune --force', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
if ("PUSH".equals(env.gitlabActionType)) {
// Currently this pipeline only runs for pushes to `develop` branch
// First clean image registry
try {
myShCmd('docker image rm oai-amf:develop', new_host_flag, new_host_user, new_host)
myShCmd('docker image rm oai-amf:develop', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
} catch (Exception e) {
echo "Maybe a previous build went wrong"
}
}
myShCmd('docker build --target oai-amf --tag oai-amf:' + amf_tag + ' --file docker/Dockerfile.ubuntu.18.04 --build-arg NEEDED_GIT_PROXY="http://proxy.eurecom.fr:8080" . > archives/amf_docker_image_build.log 2>&1', new_host_flag, new_host_user, new_host)
myShCmd('docker image ls >> archives/amf_docker_image_build.log', new_host_flag, new_host_user, new_host)
myShCmd('docker build --no-cache --target oai-amf --tag oai-amf:' + amf_tag + ' --file docker/Dockerfile.ubuntu.18.04 --build-arg NEEDED_GIT_PROXY="http://proxy.eurecom.fr:8080" . > archives/amf_docker_image_build.log 2>&1', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
myShCmd('docker image ls >> archives/amf_docker_image_build.log', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
}
}
}
post {
always {
script {
copyFrom2ndServer('archives/amf_docker_image_build.log', 'archives', new_host_flag, new_host_user, new_host)
copyFrom2ndServer('archives/amf_docker_image_build.log', 'archives', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
}
}
success {
......@@ -156,32 +178,76 @@ pipeline {
}
}
}
stage ('Build RHEL8 AMF Image') {
when { expression {rem_rhel_host_flag} }
steps {
lock (rem_rhel8_resource) {
script {
if ("PUSH".equals(env.gitlabActionType)) {
// Currently this pipeline only runs for pushes to `develop` branch
// First clean image registry
try {
myShCmd('sudo podman image rm oai-amf:develop || true', rem_rhel_host_flag, rem_rhel_host_user, rem_rhel_host)
} catch (Exception e) {
echo "Maybe a previous build went wrong"
}
else {
// In case we forgot during a previous run
myShCmd('sudo podman image rm oai-amf:' + amf_tag + ' || true', rem_rhel_host_flag, rem_rhel_host_user, rem_rhel_host)
}
myShCmd('sudo podman image prune --force', rem_rhel_host_flag, rem_rhel_host_user, rem_rhel_host)
// Copy the RHEL Host certificates for building
myShCmd('mkdir -p tmp/ca tmp/entitlement', rem_rhel_host_flag, rem_rhel_host_user, rem_rhel_host)
myShCmd('cp /etc/pki/entitlement/*pem tmp/entitlement', rem_rhel_host_flag, rem_rhel_host_user, rem_rhel_host)
myShCmd('sudo cp /etc/rhsm/ca/redhat-uep.pem tmp/ca', rem_rhel_host_flag, rem_rhel_host_user, rem_rhel_host)
myShCmd('sudo podman build --no-cache --target oai-amf --tag oai-amf:' + amf_tag + ' --file docker/Dockerfile.amf.rhel8.2 --build-arg EURECOM_PROXY="http://proxy.eurecom.fr:8080" . > archives/amf_podman_image_build.log 2>&1', rem_rhel_host_flag, rem_rhel_host_user, rem_rhel_host)
myShCmd('sudo podman image ls >> archives/amf_podman_image_build.log', rem_rhel_host_flag, rem_rhel_host_user, rem_rhel_host)
if ("PUSH".equals(env.gitlabActionType)) {
myShCmd('sudo podman image tag oai-amf:' + amf_tag + ' oai-amf:' + rhel_amf_tag, rem_rhel_host_flag, rem_rhel_host_user, rem_rhel_host)
}
}
}
}
post {
always {
script {
copyFrom2ndServer('archives/amf_podman_image_build.log', 'archives', rem_rhel_host_flag, rem_rhel_host_user, rem_rhel_host)
}
}
success {
sh "echo 'OAI-AMF PODMAN RHEL8 IMAGE BUILD: OK' >> archives/amf_podman_image_build.log"
}
unsuccessful {
sh "echo 'OAI-AMF PODMAN RHEL8 IMAGE BUILD: KO' >> archives/amf_podman_image_build.log"
}
}
}
// Running CPPCHECK in parallel to gain time
stage ('Static Code Analysis') {
steps {
script {
gitlabCommitStatus(name: "Static Code Analysis") {
// Running on xenial to have 1.72 version of cppcheck
myShCmd('docker run --name ci-cn-cppcheck -d ubuntu:xenial /bin/bash -c "sleep infinity"', new_host_flag, new_host_user, new_host)
myShCmd('docker exec ci-cn-cppcheck /bin/bash -c "apt-get update && apt-get upgrade --yes" > archives/cppcheck_install.log', new_host_flag, new_host_user, new_host)
myShCmd('docker exec ci-cn-cppcheck /bin/bash -c "apt-get install --yes cppcheck bzip2" >> archives/cppcheck_install.log', new_host_flag, new_host_user, new_host)
myShCmd('docker run --name ci-cn-cppcheck -d ubuntu:xenial /bin/bash -c "sleep infinity"', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
myShCmd('docker exec ci-cn-cppcheck /bin/bash -c "apt-get update && apt-get upgrade --yes" > archives/cppcheck_install.log', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
myShCmd('docker exec ci-cn-cppcheck /bin/bash -c "apt-get install --yes cppcheck bzip2" >> archives/cppcheck_install.log', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
myShCmd('docker cp ./openair-amf.tar.bz2 ci-cn-cppcheck:/home', new_host_flag, new_host_user, new_host)
myShCmd('docker exec ci-cn-cppcheck /bin/bash -c "cd /home && tar -xjf openair-amf.tar.bz2"', new_host_flag, new_host_user, new_host)
myShCmd('docker exec ci-cn-cppcheck /bin/bash -c "rm -f /home/openair-amf.tar.bz2"', new_host_flag, new_host_user, new_host)
myShCmd('docker cp ./openair-amf.tar.bz2 ci-cn-cppcheck:/home', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
myShCmd('docker exec ci-cn-cppcheck /bin/bash -c "cd /home && tar -xjf openair-amf.tar.bz2"', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
myShCmd('docker exec ci-cn-cppcheck /bin/bash -c "rm -f /home/openair-amf.tar.bz2"', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
myShCmd('docker exec ci-cn-cppcheck /bin/bash -c "cd /home && cppcheck --enable=warning --force --xml --xml-version=2 --suppressions-list=ci-scripts/cppcheck_suppressions.list src 2> cppcheck.xml 1> cppcheck_build.log"', new_host_flag, new_host_user, new_host)
myShCmd('docker exec ci-cn-cppcheck /bin/bash -c "cd /home && cppcheck --enable=warning --force --xml --xml-version=2 --suppressions-list=ci-scripts/cppcheck_suppressions.list src 2> cppcheck.xml 1> cppcheck_build.log"', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
}
}
}
post {
always {
script {
myShCmd('docker cp ci-cn-cppcheck:/home/cppcheck.xml archives', new_host_flag, new_host_user, new_host)
myShCmd('docker cp ci-cn-cppcheck:/home/cppcheck_build.log archives', new_host_flag, new_host_user, new_host)
copyFrom2ndServer('archives/cppcheck*.*', 'archives', new_host_flag, new_host_user, new_host)
myShCmd('docker cp ci-cn-cppcheck:/home/cppcheck.xml archives', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
myShCmd('docker cp ci-cn-cppcheck:/home/cppcheck_build.log archives', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
copyFrom2ndServer('archives/cppcheck*.*', 'archives', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
// no need to keep the cppcheck container
myShCmd('docker rm -f ci-cn-cppcheck', new_host_flag, new_host_user, new_host)
myShCmd('docker rm -f ci-cn-cppcheck', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
}
}
success {
......@@ -197,20 +263,20 @@ pipeline {
steps {
script {
gitlabCommitStatus(name: "Code Formatting Checker") {
myShCmd('docker run --name ci-cn-clang-formatter -d ubuntu:bionic /bin/bash -c "sleep infinity"', new_host_flag, new_host_user, new_host)
myShCmd('docker exec ci-cn-clang-formatter /bin/bash -c "apt-get update && apt-get upgrade --yes" > archives/clang_format_install.log', new_host_flag, new_host_user, new_host)
myShCmd('docker exec ci-cn-clang-formatter /bin/bash -c "apt-get install --yes git tree bzip2" >> archives/clang_format_install.log', new_host_flag, new_host_user, new_host)
myShCmd('docker run --name ci-cn-clang-formatter -d ubuntu:bionic /bin/bash -c "sleep infinity"', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
myShCmd('docker exec ci-cn-clang-formatter /bin/bash -c "apt-get update && apt-get upgrade --yes" > archives/clang_format_install.log', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
myShCmd('docker exec ci-cn-clang-formatter /bin/bash -c "apt-get install --yes git tree bzip2" >> archives/clang_format_install.log', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
myShCmd('docker cp ./openair-amf.tar.bz2 ci-cn-clang-formatter:/home', new_host_flag, new_host_user, new_host)
myShCmd('docker exec ci-cn-clang-formatter /bin/bash -c "cd /home && tar -xjf openair-amf.tar.bz2"', new_host_flag, new_host_user, new_host)
myShCmd('docker exec ci-cn-clang-formatter /bin/bash -c "rm -f /home/openair-amf.tar.bz2"', new_host_flag, new_host_user, new_host)
myShCmd('docker cp ./openair-amf.tar.bz2 ci-cn-clang-formatter:/home', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
myShCmd('docker exec ci-cn-clang-formatter /bin/bash -c "cd /home && tar -xjf openair-amf.tar.bz2"', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
myShCmd('docker exec ci-cn-clang-formatter /bin/bash -c "rm -f /home/openair-amf.tar.bz2"', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
// We install a dedicated version (installed on our CI server).
myShCmd('docker cp /opt/clang-format/9.0.0/bin/clang-format ci-cn-clang-formatter:/usr/local/bin', new_host_flag, new_host_user, new_host)
myShCmd('docker cp /opt/clang-format/9.0.0/bin/clang-format ci-cn-clang-formatter:/usr/local/bin', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
if ("MERGE".equals(env.gitlabActionType)) {
myShCmd('docker exec ci-cn-clang-formatter /bin/bash -c "cd /home && ./ci-scripts/checkCodingFormattingRules.sh --src-branch ' + env.gitlabSourceBranch +' --target-branch ' + env.gitlabTargetBranch + '"', new_host_flag, new_host_user, new_host)
myShCmd('docker exec ci-cn-clang-formatter /bin/bash -c "cd /home && ./ci-scripts/checkCodingFormattingRules.sh --src-branch ' + env.gitlabSourceBranch +' --target-branch ' + env.gitlabTargetBranch + '"', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
} else {
myShCmd('docker exec ci-cn-clang-formatter /bin/bash -c "cd /home && ./ci-scripts/checkCodingFormattingRules.sh"', new_host_flag, new_host_user, new_host)
myShCmd('docker exec ci-cn-clang-formatter /bin/bash -c "cd /home && ./ci-scripts/checkCodingFormattingRules.sh"', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
}
}
}
......@@ -218,17 +284,17 @@ pipeline {
post {
always {
script {
myShCmd('docker cp ci-cn-clang-formatter:/home/src/oai_rules_result.txt src', new_host_flag, new_host_user, new_host)
myShCmd('docker cp ci-cn-clang-formatter:/home/src/oai_rules_result.txt src', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
// May not have been generated
try {
myShCmd('docker cp ci-cn-clang-formatter:/home/src/oai_rules_result_list.txt src', new_host_flag, new_host_user, new_host)
myShCmd('docker cp ci-cn-clang-formatter:/home/src/oai_rules_result_list.txt src', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
} catch (Exception e) {
echo "Failed to copy src/oai_rules_result_list.txt! It may not have been generated. That's OK!"
}
copyFrom2ndServer('archives/clang_format*.*', 'archives', new_host_flag, new_host_user, new_host)
copyFrom2ndServer('src/oai_rules*.*', 'src', new_host_flag, new_host_user, new_host)
copyFrom2ndServer('archives/clang_format*.*', 'archives', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
copyFrom2ndServer('src/oai_rules*.*', 'src', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
// no need to keep the clang-formatter container
myShCmd('docker rm -f ci-cn-clang-formatter', new_host_flag, new_host_user, new_host)
myShCmd('docker rm -f ci-cn-clang-formatter', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
}
}
}
......@@ -290,16 +356,30 @@ pipeline {
// Removing temporary / intermediate images
try {
if ("MERGE".equals(env.gitlabActionType)) {
myShCmd('docker image rm --force oai-amf:ci-tmp', new_host_flag, new_host_user, new_host)
myShCmd('docker image rm --force oai-amf:ci-tmp', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
}
} catch (Exception e) {
echo "We failed to delete the OAI-AMF temp image"
}
try {
myShCmd('docker image prune --force', new_host_flag, new_host_user, new_host)
if ("MERGE".equals(env.gitlabActionType)) {
myShCmd('sudo podman image rm oai-amf:ci-temp', rem_rhel_host_flag, rem_rhel_host_user, rem_rhel_host)
}
} catch (Exception e) {
echo "We failed to delete the OAI-AMF temp image"
}
try {
myShCmd('docker image prune --force', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
} catch (Exception e) {
echo "We failed to prune all unneeded intermediate images"
}
if (rem_rhel_host_flag) {
try {
myShCmd('sudo podman image prune --force', rem_rhel_host_flag, rem_rhel_host_user, rem_rhel_host)
} catch (Exception e) {
echo "We failed to prune all unneeded intermediate images"
}
}
// Zipping all archived log files
sh "zip -r -qq docker_logs.zip archives"
......@@ -350,7 +430,7 @@ def copyTo2ndServer(filename, flag, user, host) {
sh "scp ${filename} ${user}@${host}:/tmp/CI-CN-AMF"
if ("openair-amf.tar.bz2".equals(filename)) {
sh "ssh ${user}@${host} 'cd /tmp/CI-CN-AMF && tar -xjf ${filename}'"
sh "ssh ${user}@${host} 'mkdir -p /tmp/CI-CN-AMF/archives'"
sh "ssh ${user}@${host} 'mkdir -p /tmp/CI-CN-AMF/${filename}'"
sh "ssh ${user}@${host} 'mkdir -p /tmp/CI-CN-AMF/archives/oai-amf-cfg'"
}
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment