improve test framework

git-svn-id: http://svn.eurecom.fr/openair4G/trunk@7850 818b1a75-f10b-46b9-bf7c-635c3b92a50f

cmake_targets/autotests/run_compilation_autotests.bash

@@ -7,25 +7,35 @@ else
    exit 1
 fi
 
+results_file=$tdir/log/compilation_autotests.xml
+
 # include the jUnit-like logging functions
 source $OPENAIR_DIR/cmake_targets/tools/test_helper
 
 test_compile() {
     xUnit_start
+    test_name=$1
+    compile_prog=$2
+    exec_prog=$3
+    build_dir=$tdir/$1/build
+    log_file=$tdir/log/$1.txt
+    target=$5
+    echo "Compiling test case $test_name. Log file = $log_file"
+    rm -fr $build_dir
     mkdir -p $tdir/$1/build
-    cd $tdir/$1/build
+    cd $build_dir
     {
         cmake ..
-        rm -f $3
-        make -j4 $2
-    } > $tdir/log/$1.txt 2>&1
-    if [ -s $3 ] ; then
-        cp $3 $tdir/bin/`basename $3`.$5.$1
-        echo_success "$1 $3 $5 compiled"
-        xUnit_success "compilation" $1
+        rm -f $exec_prog
+        make -j`nproc` $compile_prog
+    } > $log_file 2>&1
+    if [ -s $exec_prog ] ; then
+        cp $exec_prog $tdir/bin/`basename $exec_prog`.$target.$test_name
+        echo_success "$test_name $exec_prog $target compiled"
+        xUnit_success "compilation" $test_name "PASS" 1
     else
-        echo_error "$1 $3 $5 compilation failed"
-        xUnit_fail "compilation" $1
+        echo_error "$test_name $exec_prog $target compilation failed"
+        xUnit_fail "compilation" $test_name "FAIL" 1
     fi
 }
 
@@ -99,4 +109,6 @@ test_compile \
     rrh_gw $tdir/bin/rrh_gw
 
 # write the test results into a file
-xUnit_write "$tdir/log/compilation_autotests.xml"
+xUnit_write "$results_file"
+
+echo "Test Results are written to $results_file"

cmake_targets/autotests/run_exec_autotests.bash

 #!/bin/bash
 
-if [ -s $OPENAIR_DIR/cmake_targets/tools/build_helper ] ; then
-   source $OPENAIR_DIR/cmake_targets/tools/build_helper
+if [ -s $OPENAIR_DIR/cmake_targets/tools/test_helper ] ; then
+   source $OPENAIR_DIR/cmake_targets/tools/test_helper
 else
    echo "Error: no file in the file tree: is OPENAIR_DIR variable set?"
    exit 1
 fi
 
+
+SUDO="sudo -E "
+tdir=$OPENAIR_DIR/cmake_targets/autotests
+mkdir -p $tdir/bin $tdir/log
+results_file="$tdir/log/execution_autotests.xml"
+
+updated=$(svn st -q $OPENAIR_DIR)
+if [ "$updated" != "" ] ; then
+	echo "some files are not in svn:\n $updated"
+fi
+
+cd $tdir 
+
+
+#\param $1 -> name of test case
+#\param $2 -> name of compilation program
+#\param $3 -> arguments for compilation program
+#\param $4 -> name of pre-executable to install kernel modules, etc
+#\param $5 -> arguments of pre-executable
+#\param $6 -> name of executable
+#\param $7 -> arguments for running the program
+#\param $8 -> search expression
+#\param $9 -> number of runs
+
+test_compile_and_run() {
+    xUnit_start
+    test_case_name=$1
+    log_dir=$tdir/log
+    log_file=$tdir/log/test.$1.txt
+    compile_prog=$2
+    compile_args=$3
+    pre_exec_file=$4
+    pre_exec_args=$5
+    exec_args=$7
+    search_expr=$8
+    nruns=$9
+    build_dir=$tdir/$1/build
+    exec_file=$build_dir/$6
+    
+    #Temporary log file where execution log is stored.
+    temp_exec_log=$log_dir/temp_log.txt
+    
+    echo "Compiling test case $test_case_name. Log file = $log_file"
+
+    rm -fr $build_dir
+    mkdir -p $build_dir
+
+#    echo "log_dir = $log_dir"
+#    echo "log_file = $log_file"
+#    echo "exec_file = $exec_file"
+#    echo "args = $args"
+#    echo "search_expr = $search_expr"
+#    echo "pre_exec_file = $pre_exec_file"
+    
+    echo "<COMPILATION LOG>" > $log_file
+    cd $build_dir
+    {
+        cmake ..
+        #rm -fv $exec_file
+        make -j`nproc` $compile_prog
+    }>> $log_file 2>&1
+    echo "</COMPILATION LOG>" >> $log_file 2>&1
+
+    for (( run_index=1; run_index <= $nruns; run_index++ ))
+     do
+
+     echo "Executing test case $test_case_name, Run Index = $run_index, Log file = $log_file"
+
+     echo "-----------------------------------------------------------------------------" >> $log_file  2>&1
+     echo "<EXECUTION LOG Run = $run_index >" >> $log_file  2>&1
+ 
+
+     source $pre_exec_file $pre_exec_args >> $log_file  2>&1
+     $exec_file $exec_args > $temp_exec_log  2>&1
+     cat $temp_exec_log >> $log_file  2>&1
+     echo "</EXECUTION LOG Run = $run_index >" >> $log_file  2>&1
+
+     search_result=`grep "$search_expr" $temp_exec_log`
+
+     if [ -z "$search_result" ]; then
+        xUnit_fail "execution" "$test_case_name" "FAIL" "$run_index"
+     else
+	xUnit_success "execution" "$test_case_name" "PASS" "$run_index"
+     fi
+
+# End of for loop
+    done
+
+}
+
 dbin=$OPENAIR_DIR/cmake_targets/autotests/bin
 dlog=$OPENAIR_DIR/cmake_targets/autotests/log
 
@@ -14,6 +104,9 @@ run_test() {
 case=case$1; shift
 cmd=$1; shift
 expected=$3; shift
+echo "expected = $expected"
+exit
+
 $cmd > $dlog/$case.txt 2>&1
 if [ $expected = "true" ] ; then	 
   if $* $dlog/$case.txt; then
@@ -30,7 +123,29 @@ else
 fi
 }
 
-run_test 0200 "$dbin/oaisim.r8 -a -A AWGN -n 100" false grep -q '(Segmentation.fault)|(Exiting)|(FATAL)'
+#$1 -> name of test case
+#$2 -> name of compilation program
+#$3 -> arguments for compilation program
+#$4 -> name of pre-executable to install kernel modules, etc
+#$5 -> arguments of pre-executable
+#$6 -> name of executable
+#$7 -> arguments for running the program
+#$8 -> search expression
+#$9 -> number of runs
+
+test_compile_and_run 0200 "oaisim_nos1" "" "$OPENAIR_DIR/cmake_targets/tools/init_nas_nos1" "" "oaisim_nos1" " -O $OPENAIR_TARGETS/PROJECTS/GENERIC-LTE-EPC/CONF/enb.band7.generic.oaisim.local_no_mme.conf -b1 -u1 -n1000" "RRC_CONN" 3
+
+test_compile_and_run 0201 "oaisim_nos1" "" "$OPENAIR_DIR/cmake_targets/tools/init_nas_nos1" "" "oaisim_nos1" " -O $OPENAIR_TARGETS/PROJECTS/GENERIC-LTE-EPC/CONF/enb.band7.generic.oaisim.local_no_mme.conf -b1 -u1 -a -n1000" "RRC_CONN" 3
+
+#test_compile_and_run 0200 "oaisim_nos1" "" "$OPENAIR_DIR/cmake_targets/tools/init_nas_nos1" "" "oaisim_nos1" " -O /home/calisson/rohit/oai_snav/taets/PROJECTS/GENERIC-LTE-EPC/CONF/enb.band7.generic.oaisim.local_no_mme.conf -b1 -u1 -a " "RRC_CONN" 3
+
+#run_test 0200 "$dbin/oaisim.r8 -a -A AWGN -n 100" false grep -q '(Segmentation.fault)|(Exiting)|(FATAL)'
+
+#run_test 0201 "$dbin/oaisim.r8 -a -A AWGN -n 100" false fgrep -q '[E]'
+
+# write the test results into a file
+
+xUnit_write "$results_file"
 
-run_test 0201 "$dbin/oaisim.r8 -a -A AWGN -n 100" false fgrep -q '[E]'
+echo "Test Results are written to $results_file"
 

cmake_targets/tools/test_helper

@@ -23,11 +23,17 @@ xUnit_start() {
 # \pre xUnit_start() must have been called before
 # \param $1 classname
 # \param $2 testcase name
+# \param $3 testcase result
+# \param $4 run index
 xUnit_fail() {
+  class=$1
+  test_case=$2
+  result=$3
+  run_index=$4
   currtime=$(date +%s.%N)
   time=$(echo "$currtime - $XUNIT_START" | bc -l)
-  xml="<testcase classname='$1' name='$2' time='$time'><failure message='failed'/></testcase>"
-  XUNIT_TESTCASES_XML="$XUNIT_TESTCASES_XML $xml"
+  xml="<testcase classname='$class' name='$test_case' run='$run_index' time='$time' RESULT='$result'></testcase>"
+  XUNIT_TESTCASES_XML="$XUNIT_TESTCASES_XML \n$xml"
   XUNIT_FAILED=$((XUNIT_FAILED+1))
 }
 
@@ -36,11 +42,17 @@ xUnit_fail() {
 # \pre xUnit_start() must have been called before
 # \param $1 classname
 # \param $2 testcase name
+# \param $3 testcase result
+# \param $4 run index
 xUnit_success() {
+  class=$1
+  test_case=$2
+  result=$3
+  run_index=$4
   currtime=$(date +%s.%N)
   time=$(echo "$currtime - $XUNIT_START" | bc -l)
-  xml="<testcase classname='$1' name='$2' time='$time' />"
-  XUNIT_TESTCASES_XML="$XUNIT_TESTCASES_XML $xml"
+  xml="<testcase classname='$class' name='$test_case' run='$run_index' time='$time' RESULT='$result'></testcase>"
+  XUNIT_TESTCASES_XML="$XUNIT_TESTCASES_XML \n$xml"
   XUNIT_SUCCESS=$((XUNIT_SUCCESS+1))
 }
 
@@ -48,13 +60,14 @@ xUnit_success() {
 # This functions writes out the test report.
 # \param $1 filename
 xUnit_write() {
+  filename=$1
   tests=$((XUNIT_FAILED+XUNIT_SUCCESS))
   timestamp=$(date --iso-8601=seconds)
   time=$(echo "$currtime - $XUNIT_TESTSUITE_START" | bc -l)
   xml_header="<testsuites><testsuite errors='0' failures='$XUNIT_FAILED' hostname='$(hostname)' name='OAI' skipped='0' tests='$tests' time='$time' timestamp='$timestamp'>"
-  echo $xml_header > $1
-  echo $XUNIT_TESTCASES_XML >> $1
-  echo "</testsuite></testsuites>" >> $1
+  echo $xml_header > $filename
+  echo -e $XUNIT_TESTCASES_XML >> $filename
+  echo "</testsuite></testsuites>" >> $filename
   XUNIT_TESTSUITE_START=0
   XUNIT_START=0
   XUNIT_TOTAL=0