From 6e28bfe6eb41c2ba2b1f480caf9241f2988e1874 Mon Sep 17 00:00:00 2001 From: Raphael Defosseux <raphael.defosseux@eurecom.fr> Date: Wed, 27 Jun 2018 17:47:33 +0200 Subject: [PATCH] CI: First run of tests -- on physical simulators -- no proper report yet Signed-off-by: Raphael Defosseux <raphael.defosseux@eurecom.fr> --- ci-scripts/Jenkinsfile-gitlab | 30 +++ ci-scripts/runTestOnVM.sh | 232 ++++++++++++++++++ ci-scripts/template.xsl | 61 +++++ .../autotests/run_exec_autotests.bash | 45 +++- cmake_targets/autotests/test_case_list.xml | 4 +- 5 files changed, 360 insertions(+), 12 deletions(-) create mode 100755 ci-scripts/runTestOnVM.sh create mode 100644 ci-scripts/template.xsl diff --git a/ci-scripts/Jenkinsfile-gitlab b/ci-scripts/Jenkinsfile-gitlab index 2a29742c36..9dbb3452ef 100644 --- a/ci-scripts/Jenkinsfile-gitlab +++ b/ci-scripts/Jenkinsfile-gitlab @@ -150,6 +150,36 @@ pipeline { } } + stage ("Variant Tests") { + parallel { + stage ("Test physical simulators") { + steps { + gitlabCommitStatus(name: "Test phy-sim") { + sh "./ci-scripts/runTestOnVM.sh --workspace $WORKSPACE --variant phy-sim --job-name ${JOB_NAME} --build-id ${BUILD_ID} --keep-vm-alive" + } + } + } + stage ("Test basic simulator") { + steps { + //gitlabCommitStatus(name: "Test basic-sim") { + sh "./ci-scripts/runTestOnVM.sh --workspace $WORKSPACE --variant phy-sim --job-name ${JOB_NAME} --build-id ${BUILD_ID} --keep-vm-alive" + //} + } + } + } + post { + always { + script { + dir ('archives') { + sh "zip -r -qq vm_tests_logs.zip */test" + } + if(fileExists('archives/vm_tests_logs.zip')) { + archiveArtifacts artifacts: 'archives/vm_tests_logs.zip' + } + } + } + } + } stage ("Destroy all Virtual Machines") { steps { sh "./ci-scripts/destroyAllRunningVM.sh --job-name ${JOB_NAME} --build-id ${BUILD_ID}" diff --git a/ci-scripts/runTestOnVM.sh b/ci-scripts/runTestOnVM.sh new file mode 100755 index 0000000000..1edc45a996 --- /dev/null +++ b/ci-scripts/runTestOnVM.sh @@ -0,0 +1,232 @@ +#!/bin/bash + +function usage { + echo "OAI VM Test Run script" + echo " Original Author: Raphael Defosseux" + echo " Default:" + echo " -- eNB with USRP" + echo "" + echo "Usage:" + echo "------" + echo " runTestOnVM.sh [OPTIONS]" + echo "" + echo "Options:" + echo "--------" + echo " --job-name #### OR -jn ####" + echo " Specify the name of the Jenkins job." + echo "" + echo " --build-id #### OR -id ####" + echo " Specify the build ID of the Jenkins job." + echo "" + echo " --workspace #### OR -ws ####" + echo " Specify the workspace." + echo "" + echo " --variant enb-usrp OR -v1" + echo " --variant basic-sim OR -v2" + echo " --variant phy-sim OR -v3" + echo " --variant cppcheck OR -v4" + echo " --variant enb-ethernet OR -v7" + echo " --variant ue-ethernet OR -v8" + echo " Specify the variant to build." + echo "" + echo " --keep-vm-alive OR -k" + echo " Keep the VM alive after the build." + echo "" + echo " --help OR -h" + echo " Print this help message." + echo "" +} + +function variant_usage { + echo "OAI VM Test Run script" + echo " Original Author: Raphael Defosseux" + echo "" + echo " --variant enb-usrp OR -v1" + echo " --variant basic-sim OR -v2" + echo " --variant phy-sim OR -v3" + echo " --variant cppcheck OR -v4" + echo " --variant enb-ethernet OR -v7" + echo " --variant ue-ethernet OR -v8" + echo "" +} + +if [ $# -lt 1 ] || [ $# -gt 9 ] +then + echo "Syntax Error: not the correct number of arguments" + echo "" + usage + exit 1 +fi + +VM_TEMPLATE=ci- +JOB_NAME=XX +BUILD_ID=XX +VM_NAME=ci-enb-usrp +ARCHIVES_LOC=enb_usrp/test +KEEP_VM_ALIVE=0 +RUN_OPTIONS="none" + +while [[ $# -gt 0 ]] +do +key="$1" + +case $key in + -h|--help) + shift + usage + exit 0 + ;; + -jn|--job-name) + JOB_NAME="$2" + shift + shift + ;; + -id|--build-id) + BUILD_ID="$2" + shift + shift + ;; + -ws|--workspace) + JENKINS_WKSP="$2" + shift + shift + ;; + -k|--keep-vm-alive) + KEEP_VM_ALIVE=1 + shift + ;; + -v1) + VM_NAME=ci-enb-usrp + shift + ;; + -v2) + VM_NAME=ci-basic-sim + shift + ;; + -v3) + VM_NAME=ci-phy-sim + RUN_OPTIONS="./run_exec_autotests.bash -g \"01510*\" -q -np -b" + ARCHIVES_LOC=phy_sim/test + shift + ;; + -v4) + VM_NAME=ci-cppcheck + shift + ;; + -v7) + VM_NAME=ci-enb-ethernet + shift + ;; + -v8) + VM_NAME=ci-ue-ethernet + shift + ;; + --variant) + variant="$2" + case $variant in + enb-usrp) + VM_NAME=ci-enb-usrp + ;; + basic-sim) + VM_NAME=ci-basic-sim + ;; + phy-sim) + VM_NAME=ci-phy-sim + ;; + cppcheck) + VM_NAME=ci-cppcheck + ;; + enb-ethernet) + VM_NAME=ci-enb-ethernet + ;; + ue-ethernet) + VM_NAME=ci-ue-ethernet + ;; + *) + echo "" + echo "Syntax Error: Invalid Variant option -> $variant" + echo "" + variant_usage + exit 1 + esac + shift + shift + ;; + *) + echo "Syntax Error: unknown option: $key" + echo "" + usage + exit 1 +esac +done + +if [ "$JOB_NAME" == "XX" ] || [ "$BUILD_ID" == "XX" ] +then + VM_TEMPLATE=ci- +else + VM_TEMPLATE=${JOB_NAME}-b${BUILD_ID}- +fi + +VM_NAME=`echo $VM_NAME | sed -e "s#ci-#$VM_TEMPLATE#"` +VM_CMDS=${VM_NAME}_cmds.txt +ARCHIVES_LOC=${JENKINS_WKSP}/archives/${ARCHIVES_LOC} + +echo "VM_NAME = $VM_NAME" +echo "VM_CMD_FILE = $VM_CMDS" +echo "JENKINS_WKSP = $JENKINS_WKSP" +echo "ARCHIVES_LOC = $ARCHIVES_LOC" + +echo "############################################################" +echo "Waiting for VM to be started" +echo "############################################################" +uvt-kvm wait $VM_NAME --insecure + +VM_IP_ADDR=`uvt-kvm ip $VM_NAME` +echo "$VM_NAME has for IP addr = $VM_IP_ADDR" + +if [ "$RUN_OPTIONS" == "none" ] +then + echo "No run on VM testing for this variant currently" +else + + echo "############################################################" + echo "Running test script on VM ($VM_NAME)" + echo "############################################################" + echo "echo \"sudo apt-get --yes --quiet install bc \"" > $VM_CMDS + echo "sudo apt-get update > bc-install.txt 2>&1" >> $VM_CMDS + echo "sudo apt-get --yes install bc >> bc-install.txt 2>&1" >> $VM_CMDS + echo "cd tmp" >> $VM_CMDS + echo "echo \"source oaienv\"" >> $VM_CMDS + echo "source oaienv" >> $VM_CMDS + echo "echo \"cd cmake_targets/autotests\"" >> $VM_CMDS + echo "cd cmake_targets/autotests" >> $VM_CMDS + echo "echo \"rm -Rf log\"" >> $VM_CMDS + echo "rm -Rf log" >> $VM_CMDS + echo "$RUN_OPTIONS" | sed -e 's@"@\\"@g' -e 's@^@echo "@' -e 's@$@"@' >> $VM_CMDS + echo "$RUN_OPTIONS" >> $VM_CMDS + echo "cp /home/ubuntu/bc-install.txt log" >> $VM_CMDS + + ssh -o StrictHostKeyChecking=no ubuntu@$VM_IP_ADDR < $VM_CMDS + + echo "############################################################" + echo "Creating a tmp folder to store results and artifacts" + echo "############################################################" + + if [ -d $ARCHIVES_LOC ] + then + rm -Rf $ARCHIVES_LOC + fi + scp -rf -o StrictHostKeyChecking=no ubuntu@$VM_IP_ADDR:/home/ubuntu/tmp/cmake_targets/autotests/log $ARCHIVES_LOC + + if [ $KEEP_VM_ALIVE -eq 0 ] + then + echo "############################################################" + echo "Destroying VM" + echo "############################################################" + uvt-kvm destroy $VM_NAME + ssh-keygen -R $VM_IP_ADDR + fi + rm -f $VM_CMDS +fi + +exit 0 diff --git a/ci-scripts/template.xsl b/ci-scripts/template.xsl new file mode 100644 index 0000000000..64fb23e4cd --- /dev/null +++ b/ci-scripts/template.xsl @@ -0,0 +1,61 @@ +<?xml version="1.0" encoding="utf-8"?> +<xsl:stylesheet version="2.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform"> + <xsl:output method="html" version="4" encoding="UTF-8" indent="no" omit-xml-declaration="yes"/> + <!-- main body --> + <xsl:template match="/"> + <html> + <body> + <h3>TEMPLATE Results Summary</h3> + <table border="1"> + <tr bgcolor="lightcyan"> + <!--Header only so select first row to get headers--> + <th>Hostname</th> + <th>Nb Tests</th> + <th>Failures</th> + <th>Timestamp</th> + </tr> + <!--Get all the other rows--> + <xsl:for-each select="testsuites/testsuite"> + <tr> + <td> + <xsl:value-of select="@hostname"/> + </td> + <td> + <xsl:value-of select="@tests"/> + </td> + <td> + <xsl:value-of select="@failures"/> + </td> + <td> + <xsl:value-of select="@timestamp"/> + </td> + </tr> + </xsl:for-each> + </table> + <h4>Details</h4> + <table border="1"> + <tr bgcolor="lightcyan"> + <!--Header only so select first row to get headers--> + <th>Test Name</th> + <th>Result</th> + <th>Time</th> + </tr> + <!--Get all the other rows--> + <xsl:for-each select="testsuites/testsuite/testcase"> + <tr> + <td> + <xsl:value-of select="@name"/> + </td> + <td> + <xsl:value-of select="@RESULT"/> + </td> + <td> + <xsl:value-of select="@time"/> + </td> + </tr> + </xsl:for-each> + </table> + </body> + </html> + </xsl:template> +</xsl:stylesheet> diff --git a/cmake_targets/autotests/run_exec_autotests.bash b/cmake_targets/autotests/run_exec_autotests.bash index ff38ae9d3c..6c5b9193b6 100755 --- a/cmake_targets/autotests/run_exec_autotests.bash +++ b/cmake_targets/autotests/run_exec_autotests.bash @@ -200,6 +200,7 @@ function test_compile() { #\param $14 -> tags to help identify the test case for readability in output xml file #\param $15 => password for the user to run certain commands as sudo #\param $16 => test config file params to be modified +#\param $17 => bypass flag if main_exec if available function test_compile_and_run() { xUnit_start @@ -221,6 +222,7 @@ function test_compile_and_run() { tags=${14} mypassword=${15} test_config_file=${16} + bypass_compile=${17} build_dir=$tdir/$1/build #exec_file=$build_dir/$6 @@ -231,8 +233,6 @@ function test_compile_and_run() { rm -fr $log_dir mkdir -p $log_dir - rm -fr $OPENAIR_DIR/cmake_targets/log - echo "" > $temp_exec_log echo "" > $log_file #echo "log_dir = $log_dir" @@ -257,7 +257,13 @@ function test_compile_and_run() { main_exec_args_array=() readarray -t main_exec_args_array <<< "$exec_args" - + REAL_MAIN_EXEC=`eval "echo $main_exec"` + if [ "$bypass_compile" == "1" ] && [ -f $REAL_MAIN_EXEC ] + then + echo "Bypassing compilation for $main_exec" + else + rm -fr $OPENAIR_DIR/cmake_targets/log + #for search_expr in "${compile_prog_array[@]}" #do echo "Compiling test case $test_case_name Log file = $log_file" @@ -283,6 +289,7 @@ function test_compile_and_run() { }>> $log_file 2>&1 echo "</COMPILATION LOG>" >> $log_file 2>&1 #done + fi #process the test case if it is that of execution if [ "$class" == "execution" ]; then @@ -393,10 +400,18 @@ Options Run test cases in a group. For example, ./run_exec_autotests "0101* 010102" -p Use password for logging +-np | --no-password + No need for a password +-q | --quiet + Quiet mode; eliminate informational messages and comment prompts. +-b | --bypass-compile + Bypass compilation of main-exec if already present ' } function main () { +QUIET=0 +BYPASS_COMPILE=0 RUN_GROUP=0 SET_PASSWORD=0 passwd="" @@ -419,6 +434,16 @@ until [ -z "$1" ] SET_PASSWORD=1 passwd=$2 shift 2;; + -np|--no-password) + SET_PASSWORD=1 + shift ;; + -q|--quiet) + QUIET=1 + shift ;; + -b|--bypass-compile) + BYPASS_COMPILE=1 + echo "bypass option ON" + shift ;; -h | --help) print_help exit 1;; @@ -449,15 +474,15 @@ xml_conf="$OPENAIR_DIR/cmake_targets/autotests/test_case_list.xml" test_case_list=`xmlstarlet sel -T -t -m /testCaseList/testCase -s A:N:- "@id" -v "@id" -n $xml_conf` test_case_excl_list=`xmlstarlet sel -t -v "/testCaseList/TestCaseExclusionList" $xml_conf` -echo "Test Case Exclusion List = $test_case_excl_list " +if [ $QUIET -eq 0 ]; then echo "Test Case Exclusion List = $test_case_excl_list "; fi test_case_excl_list=`sed "s/\+/\*/g" <<< "$test_case_excl_list" ` # Replace + with * for bash string substituion read -a test_case_excl_array <<< "$test_case_excl_list" -echo "test_case_list = $test_case_list" +if [ $QUIET -eq 0 ]; then echo "test_case_list = $test_case_list"; fi -echo "Test Case Exclusion List = $test_case_excl_list \n" +if [ $QUIET -eq 0 ]; then echo "Test Case Exclusion List = $test_case_excl_list \n"; fi readarray -t test_case_array <<<"$test_case_list" @@ -484,7 +509,7 @@ for search_expr in "${test_case_array[@]}" do if [[ $search_expr == $search_excl ]];then flag_run_test_case=0 - echo_info "Test case $search_expr match found in test case excl group. Will skip the test case for execution..." + if [ $QUIET -eq 0 ]; then echo_info "Test case $search_expr match found in test case excl group. Will skip the test case for execution..."; fi break fi done @@ -533,8 +558,8 @@ for search_expr in "${test_case_array[@]}" search_array_true=() - IFS=\" #set the shell's field separator - set -f #don't try to glob + IFS=\" #set the shell field separator + set -f #dont try to glob #set -- $search_expr_true #split on $IFS for i in $search_expr_true do echo "i = $i" @@ -551,7 +576,7 @@ for search_expr in "${test_case_array[@]}" test_compile "$name" "$compile_prog" "$compile_prog_args" "$pre_exec" "$pre_exec_args" "$main_exec" "$main_exec_args" "search_array_true[@]" "$search_expr_false" "$nruns" "$pre_compile_prog" "$class" "$compile_prog_out" "$tags" elif [ "$class" == "execution" ]; then echo \'passwd\' | $SUDO killall -q oaisim_nos1 - test_compile_and_run "$name" "$compile_prog" "$compile_prog_args" "$pre_exec" "$pre_exec_args" "$main_exec" "$main_exec_args" "search_array_true[@]" "$search_expr_false" "$nruns" "$pre_compile_prog" "$class" "$compile_prog_out" "$tags" "$mypassword" "$test_config_file" + test_compile_and_run "$name" "$compile_prog" "$compile_prog_args" "$pre_exec" "$pre_exec_args" "$main_exec" "$main_exec_args" "search_array_true[@]" "$search_expr_false" "$nruns" "$pre_compile_prog" "$class" "$compile_prog_out" "$tags" "$mypassword" "$test_config_file" "$BYPASS_COMPILE" else echo "Unexpected class of test case...Skipping the test case $name ...." fi diff --git a/cmake_targets/autotests/test_case_list.xml b/cmake_targets/autotests/test_case_list.xml index d9ed38a4ec..27f43f6f2f 100644 --- a/cmake_targets/autotests/test_case_list.xml +++ b/cmake_targets/autotests/test_case_list.xml @@ -955,7 +955,7 @@ <compile_prog_args> --phy_simulators -c </compile_prog_args> <pre_exec>$OPENAIR_DIR/cmake_targets/autotests/tools/free_mem.bash</pre_exec> <pre_exec_args></pre_exec_args> - <main_exec> $OPENAIR_DIR/cmake_targets/lte-simulators/build/dlsim</main_exec> + <main_exec> $OPENAIR_DIR/targets/bin/dlsim.Rel14</main_exec> <main_exec_args> -m5 -gF -s-1 -w1.0 -f.2 -n500 -B50 -c2 -z2 -O70 -m4 -gF -s0 -w1.0 -f.2 -n500 -B6 -c4 -z2 -O70 -m15 -gF -s6.7 -w1.0 -f.2 -n500 -B50 -c2 -z2 -O70 @@ -989,7 +989,7 @@ <compile_prog_args> --phy_simulators -c </compile_prog_args> <pre_exec>$OPENAIR_DIR/cmake_targets/autotests/tools/free_mem.bash</pre_exec> <pre_exec_args></pre_exec_args> - <main_exec> $OPENAIR_DIR/cmake_targets/lte-simulators/build/ulsim</main_exec> + <main_exec> $OPENAIR_DIR/targets/bin/ulsim.Rel14</main_exec> <main_exec_args> -B25 -m5 -y1 -gN -x1 -s6 -w1.0 -e.1 -P -n500 -O70 -L -B25 -m16 -y1 -gN -x1 -s12 -w1.0 -e.1 -P -n500 -O70 -L -B50 -m5 -y1 -gN -x1 -s6 -w1.0 -e.1 -P -n500 -O70 -L -- 2.26.2