CI: transition from VM-based phy-simulator testing

    to Cluster-based testing

- In old VM pipeline, disabled phy-sim VM creation, build and testing
- In new Container pipeline, added new stage
Signed-off-by: Raphael Defosseux <raphael.defosseux@eurecom.fr>

ci-scripts/Jenkinsfile-GitLab-Container

@@ -176,6 +176,30 @@ pipeline {
         }
       }
     }
+    stage ("Image Test Processes") {
+      parallel {
+        stage ("Physical Simulators") {
+          when { expression {doFlexranCtrlTest && doMandatoryTests} }
+          steps {
+            script {
+              triggerSlaveJob ('RAN-PhySim-Cluster', 'Test-Physim-Cluster')
+            }
+          }
+          post {
+            always {
+              script {
+                finalizeSlaveJob('RAN-PhySim-Cluster')
+              }
+            }
+            failure {
+              script {
+                currentBuild.result = 'FAILURE'
+              }
+            }
+          }
+        }
+      }
+    }
   }
   post {
     always {

ci-scripts/Jenkinsfile-gitlab

@@ -196,16 +196,6 @@ pipeline {
             }
         }
 
-        stage ("Start VM -- phy-sim") {
-            steps {
-              lock (vmResource) {
-                timeout (time: 5, unit: 'MINUTES') {
-                    sh "./ci-scripts/oai-ci-vm-tool build --workspace $WORKSPACE --variant phy-sim --job-name ${JOB_NAME} --build-id ${BUILD_ID} --daemon"
-                }
-              }
-            }
-        }
-        
         stage ("Start VM -- enb-ethernet") {
             steps {
               lock (vmResource) {
@@ -255,15 +245,6 @@ pipeline {
                         }
                     }
                 }
-                stage ("Build physical simulators") {
-                    steps {
-                        gitlabCommitStatus(name: "Build phy-sim") {
-                            timeout (time: 30, unit: 'MINUTES') {
-                               sh "./ci-scripts/oai-ci-vm-tool wait --workspace $WORKSPACE --variant phy-sim --job-name ${JOB_NAME} --build-id ${BUILD_ID} --keep-vm-alive"
-                            }
-                        }
-                    }
-                }
                 stage ("Build eNB-ethernet") {
                     steps {
                         gitlabCommitStatus(name: "Build eNB-ethernet") {
@@ -296,9 +277,9 @@ pipeline {
                     script {
                         dir ('archives') {
                             if (fileExists('red_hat')) {
-                                sh "zip -r -qq vm_build_logs.zip basic_sim phy_sim enb_eth ue_eth gnb_usrp nr_ue_usrp red_hat"
+                                sh "zip -r -qq vm_build_logs.zip basic_sim enb_eth ue_eth gnb_usrp nr_ue_usrp red_hat"
                             } else {
-                                sh "zip -r -qq vm_build_logs.zip basic_sim phy_sim enb_eth ue_eth gnb_usrp nr_ue_usrp"
+                                sh "zip -r -qq vm_build_logs.zip basic_sim enb_eth ue_eth gnb_usrp nr_ue_usrp"
                             }
                         }
                         if(fileExists('archives/vm_build_logs.zip')) {
@@ -332,26 +313,6 @@ pipeline {
             parallel {
                 stage ("VM-based tests") {
                     stages {
-                        stage ("Test physical simulators") {
-                            when {
-                                expression {doMandatoryTests}
-                            }
-                            steps {
-                              lock (vmResource) {
-                                script {
-                                    timeout (time: 90, unit: 'MINUTES') {
-                                        try {
-                                            gitlabCommitStatus(name: "Test phy-sim") {
-                                                sh "./ci-scripts/oai-ci-vm-tool test --workspace $WORKSPACE --variant phy-sim --job-name ${JOB_NAME} --build-id ${BUILD_ID}"
-                                            }
-                                        } catch (Exception e) {
-                                          currentBuild.result = 'FAILURE'
-                                        }
-                                    }
-                                }
-                              }
-                            }
-                        }
                         stage ("Build Flexran Controller") {
                             when {
                                 expression {doFlexranCtrlTest && doMandatoryTests}
@@ -716,7 +677,7 @@ pipeline {
                     script {
                       if (doMandatoryTests) {
                         dir ('archives') {
-                            sh "if [ -d basic_sim/test ] || [ -d phy_sim/test ] || [ -d l2_sim/test ]; then zip -r -qq vm_tests_logs.zip */test ; fi"
+                            sh "if [ -d basic_sim/test ] || [ -d rf_sim/test ] || [ -d l2_sim/test ]; then zip -r -qq vm_tests_logs.zip */test ; fi"
                         }
                         if(fileExists('archives/vm_tests_logs.zip')) {
                             archiveArtifacts artifacts: 'archives/vm_tests_logs.zip'

ci-scripts/buildOnVM.sh

@@ -62,7 +62,7 @@ function build_on_vm {
     echo "ARCHIVES_LOC        = $ARCHIVES_LOC"
     echo "BUILD_OPTIONS       = $BUILD_OPTIONS"
 
-    if [[ "$VM_NAME" == *"-enb-usrp"* ]] || [[ "$VM_NAME" == *"-cppcheck"* ]]
+    if [[ "$VM_NAME" == *"-enb-usrp"* ]] || [[ "$VM_NAME" == *"-cppcheck"* ]] || [[ "$VM_NAME" == *"-phy-sim"* ]]
     then
         echo "This VM type is no longer supported in the pipeline framework"
         return

ci-scripts/createVM.sh

@@ -87,7 +87,7 @@ function create_vm {
     echo "VM_CPU              = $VM_CPU"
     echo "VM_DISK             = $VM_DISK GBytes"
 
-    if [[ "$VM_NAME" == *"-enb-usrp"* ]] || [[ "$VM_NAME" == *"-cppcheck"* ]]
+    if [[ "$VM_NAME" == *"-enb-usrp"* ]] || [[ "$VM_NAME" == *"-cppcheck"* ]] || [[ "$VM_NAME" == *"-phy-sim"* ]]
     then
         echo "This VM type is no longer supported in the pipeline framework"
         return

ci-scripts/reportBuildLocally.sh

@@ -568,33 +568,6 @@ function report_build {
 
     echo "   <h2>Ubuntu 16.04 LTS -- Summary</h2>" >> ./build_results.html
 
-    if [ -f ./archives/cppcheck/cppcheck.xml ]
-    then
-        sca_summary_table_header ./archives/cppcheck/cppcheck.xml "OAI Static Code Analysis with CPPCHECK"
-        sca_summary_table_row ./archives/cppcheck/cppcheck.xml "Uninitialized variable" uninitvar
-        sca_summary_table_row ./archives/cppcheck/cppcheck.xml "Uninitialized struct member" uninitStructMember
-        sca_summary_table_row ./archives/cppcheck/cppcheck.xml "Memory leak" memleak
-        sca_summary_table_row ./archives/cppcheck/cppcheck.xml "Memory is freed twice" doubleFree
-        sca_summary_table_row ./archives/cppcheck/cppcheck.xml "Resource leak" resourceLeak
-        sca_summary_table_row ./archives/cppcheck/cppcheck.xml "Possible null pointer dereference" nullPointer
-        sca_summary_table_row ./archives/cppcheck/cppcheck.xml "Array access  out of bounds" arrayIndexOutOfBounds
-        sca_summary_table_row ./archives/cppcheck/cppcheck.xml "Buffer is accessed out of bounds" bufferAccessOutOfBounds
-        sca_summary_table_row ./archives/cppcheck/cppcheck.xml "Expression depends on order of evaluation of side effects" unknownEvaluationOrder
-        sca_summary_table_footer ./archives/cppcheck/cppcheck.xml
-    fi
-
-#    summary_table_header "OAI Build: 4G LTE eNB -- USRP option" ./archives/enb_usrp
-#    summary_table_row "LTE SoftModem - Release 15" ./archives/enb_usrp/lte-softmodem.Rel15.txt "Built target lte-softmodem" ./enb_usrp_row1.html
-#    summary_table_row "Coding - Release 15" ./archives/enb_usrp/coding.Rel15.txt "Built target coding" ./enb_usrp_row2.html
-#    summary_table_row "OAI USRP device if - Release 15" ./archives/enb_usrp/oai_usrpdevif.Rel15.txt "Built target oai_usrpdevif" ./enb_usrp_row3.html
-#    summary_table_row "OAI ETHERNET transport - Release 15" ./archives/enb_usrp/oai_eth_transpro.Rel15.txt "Built target oai_eth_transpro" ./enb_usrp_row4.html
-#    summary_table_row "Parameters Lib Config - Release 15" ./archives/enb_usrp/params_libconfig.Rel15.txt "Built target params_libconfig" ./enb_usrp_row5.html
-#    summary_table_row "NASMESH - Release 15" ./archives/enb_usrp/nasmesh.Rel15.txt "Built target nasmesh" ./enb_usrp_row6.html
-#    summary_table_row "RB Tool - Release 15" ./archives/enb_usrp/rb_tool.Rel15.txt "Built target rb_tool" ./enb_usrp_row7.html
-#    summary_table_row "RF Simulator - Release 15" ./archives/enb_usrp/rfsimulator.Rel15.txt "Built target rfsimulator" ./enb_usrp_row8.html
-#    summary_table_row "TCP Bridge - Release 15" ./archives/enb_usrp/tcp_bridge_oai.Rel15.txt "Built target tcp_bridge_oai" ./enb_usrp_row9.html
-#    summary_table_footer
-#
     summary_table_header "OAI Build: 4G LTE eNB -- USRP option" ./archives/enb_eth
     summary_table_row "LTE SoftModem - Release 15" ./archives/enb_eth/lte-softmodem.Rel15.txt "Built target lte-softmodem" ./enb_eth_row1.html
     summary_table_row "Coding - Release 15" ./archives/enb_eth/coding.Rel15.txt "Built target coding" ./enb_eth_row2.html
@@ -637,48 +610,6 @@ function report_build {
     summary_table_row "NVRAM - Release 15" ./archives/basic_sim/nvram.Rel15.txt "Built target nvram" ./basic_sim_row13.html
     summary_table_footer
 
-    summary_table_header "OAI Build: 4G LTE / 5G NR Physical simulators option" ./archives/phy_sim
-    summary_table_row "LTE DL Simulator - Release 15" ./archives/phy_sim/dlsim.Rel15.txt "Built target dlsim" ./phy_sim_row1.html
-    summary_table_row "LTE UL Simulator - Release 15" ./archives/phy_sim/ulsim.Rel15.txt "Built target ulsim" ./phy_sim_row2.html
-    summary_table_row "Coding - Release 15" ./archives/phy_sim/coding.Rel15.txt "Built target coding" ./phy_sim_row3.html
-    if [ -f ./archives/phy_sim/ldpctest.Rel15.txt ]
-    then
-        summary_table_row "NR LDPC Test - Release 15" ./archives/phy_sim/ldpctest.Rel15.txt "Built target ldpctest" ./phy_sim_row4.html
-    fi
-    if [ -f ./archives/phy_sim/polartest.Rel15.txt ]
-    then
-        summary_table_row "NR Polar Test - Release 15" ./archives/phy_sim/polartest.Rel15.txt "Built target polartest" ./phy_sim_row5.html
-    fi
-    if [ -f ./archives/phy_sim/nr_pbchsim.Rel15.txt ]
-    then
-        summary_table_row "NR PBCH Test - Release 15" ./archives/phy_sim/nr_pbchsim.Rel15.txt "Built target nr_pbchsim" ./phy_sim_row6.html
-    fi
-    if [ -f ./archives/phy_sim/nr_dlschsim.Rel15.txt ]
-    then
-        summary_table_row "NR DLSCH Test - Release 15" ./archives/phy_sim/nr_dlschsim.Rel15.txt "Built target nr_dlschsim" ./phy_sim_row7.html
-    fi
-    if [ -f ./archives/phy_sim/nr_pucchsim.Rel15.txt ]
-    then
-        summary_table_row "NR PUCCH Test - Release 15" ./archives/phy_sim/nr_pucchsim.Rel15.txt "Built target nr_pucchsim" ./phy_sim_row8.html
-    fi
-    if [ -f ./archives/phy_sim/smallblocktest.Rel15.txt ]
-    then
-        summary_table_row "NR Small Block Test - Release 15" ./archives/phy_sim/smallblocktest.Rel15.txt "Built target smallblocktest" ./phy_sim_row9.html
-    fi
-    if [ -f ./archives/phy_sim/nr_dlsim.Rel15.txt ]
-    then
-        summary_table_row "NR DL Sim Test - Release 15" ./archives/phy_sim/nr_dlsim.Rel15.txt "Built target nr_dlsim" ./phy_sim_row10.html
-    fi
-    if [ -f ./archives/phy_sim/nr_ulschsim.Rel15.txt ]
-    then
-        summary_table_row "NR ULSCH Test - Release 15" ./archives/phy_sim/nr_ulschsim.Rel15.txt "Built target nr_ulschsim" ./phy_sim_row11.html
-    fi
-    if [ -f ./archives/phy_sim/nr_ulsim.Rel15.txt ]
-    then
-        summary_table_row "NR UL Sim Test - Release 15" ./archives/phy_sim/nr_ulsim.Rel15.txt "Built target nr_ulsim" ./phy_sim_row12.html
-    fi
-    summary_table_footer
-
     if [ -f archives/gnb_usrp/nr-softmodem.Rel15.txt ]
     then
         summary_table_header "OAI Build: 5G NR gNB -- USRP option" ./archives/gnb_usrp
@@ -721,13 +652,6 @@ function report_build {
     echo "   <button data-toggle=\"collapse\" data-target=\"#oai-compilation-details\">Details for Compilation Errors and Warnings </button>" >> ./build_results.html
     echo "   <div id=\"oai-compilation-details\" class=\"collapse\">" >> ./build_results.html
 
-#    if [ -f ./enb_usrp_row1.html ] || [ -f ./enb_usrp_row2.html ] || [ -f ./enb_usrp_row3.html ] || [ -f ./enb_usrp_row4.html ]
-#    then
-#        for DETAILS_TABLE in `ls ./enb_usrp_row*.html`
-#        do
-#            cat $DETAILS_TABLE >> ./build_results.html
-#        done
-#    fi
     if [ -f ./enb_eth_row1.html ] || [ -f ./enb_eth_row2.html ] || [ -f ./enb_eth_row3.html ] || [ -f ./enb_eth_row4.html ] || [ -f ./enb_eth_row5.html ] || [ -f ./enb_eth_row6.html ] || [ -f ./enb_eth_row7.html ] || [ -f ./enb_eth_row8.html ] || [ -f ./enb_eth_row9.html ]
     then
         for DETAILS_TABLE in `ls ./enb_eth_row*.html`
@@ -749,13 +673,6 @@ function report_build {
             cat $DETAILS_TABLE >> ./build_results.html
         done
     fi
-    if [ -f ./phy_sim_row1.html ] || [ -f ./phy_sim_row2.html ] || [ -f ./phy_sim_row3.html ] || [ -f ./phy_sim_row4.html ] || [ -f ./phy_sim_row5.html ] || [ -f ./phy_sim_row6.html ] || [ -f ./phy_sim_row7.html ] || [ -f ./phy_sim_row8.html ]
-    then
-        for DETAILS_TABLE in `ls ./phy_sim_row*.html`
-        do
-            cat $DETAILS_TABLE >> ./build_results.html
-        done
-    fi
     if [ -f ./gnb_usrp_row1.html ] || [ -f ./gnb_usrp_row2.html ] || [ -f ./gnb_usrp_row3.html ] || [ -f ./gnb_usrp_row4.html ]
     then 
         for DETAILS_TABLE in `ls ./gnb_usrp_row*.html`
@@ -770,13 +687,6 @@ function report_build {
             cat $DETAILS_TABLE >> ./build_results.html
         done
     fi
-    if [ -f ./enb_usrp_rh_row1.html ] || [ -f ./enb_usrp_rh_row2.html ] || [ -f ./enb_usrp_rh_row3.html ] || [ -f ./enb_usrp_rh_row4.html ]
-    then
-        for DETAILS_TABLE in `ls ./enb_usrp_rh_row*.html`
-        do
-            cat $DETAILS_TABLE >> ./build_results.html
-        done
-    fi
     rm -f ./*_row*.html
 
     echo "   </div>" >> ./build_results.html

ci-scripts/runTestOnVM.sh

@@ -1478,13 +1478,13 @@ function run_test_on_vm {
         echo "$VM_NAME has for IP addr = $VM_IP_ADDR"
     fi
 
-    if [ "$RUN_OPTIONS" == "none" ]
+    if [ "$RUN_OPTIONS" == "none" ] || [[ $RUN_OPTIONS =~ .*run_exec_autotests.* ]]
     then
         echo "No run on VM testing for this variant currently"
         return
     fi
 
-    if [[ $RUN_OPTIONS =~ .*run_exec_autotests.* ]]
+    if [[ $RUN_OPTIONS =~ .*run_XXXX_autotests.* ]]
     then
         echo "############################################################"
         echo "Running test script on VM ($VM_NAME)"

ci-scripts/waitBuildOnVM.sh

@@ -47,7 +47,7 @@ function wait_on_vm_build {
     echo "ARCHIVES_LOC        = $ARCHIVES_LOC"
     echo "BUILD_OPTIONS       = $BUILD_OPTIONS"
 
-    if [[ "$VM_NAME" == *"-enb-usrp"* ]] || [[ "$VM_NAME" == *"-cppcheck"* ]]
+    if [[ "$VM_NAME" == *"-enb-usrp"* ]] || [[ "$VM_NAME" == *"-cppcheck"* ]] || [[ "$VM_NAME" == *"-phy-sim"* ]]
     then
         echo "This VM type is no longer supported in the pipeline framework"
         return
@@ -93,7 +93,7 @@ function wait_on_vm_build {
 }
 
 function check_on_vm_build {
-    if [[ "$VM_NAME" == *"-enb-usrp"* ]] || [[ "$VM_NAME" == *"-cppcheck"* ]]
+    if [[ "$VM_NAME" == *"-enb-usrp"* ]] || [[ "$VM_NAME" == *"-cppcheck"* ]] || [[ "$VM_NAME" == *"-phy-sim"* ]]
     then
         echo "This VM type is no longer supported in the pipeline framework"
         return