CI: enhancements with timeouts and build timestamps in HTML reports

Signed-off-by: Raphael Defosseux <raphael.defosseux@eurecom.fr>

ci-scripts/Jenkinsfile-gitlab

@@ -48,6 +48,9 @@ pipeline {
         stage ("Verify Parameters") {
             steps {
                 script {
+                    JOB_TIMESTAMP = sh returnStdout: true, script: 'date --utc --rfc-3339=seconds | sed -e "s#+00:00##"'
+                    JOB_TIMESTAMP = JOB_TIMESTAMP.trim()
+
                     echo '\u2705 \u001B[32mVerify Parameters\u001B[0m'
                     def allParametersPresent = true
 
@@ -114,84 +117,108 @@ pipeline {
 
         stage ("Start VM -- cppcheck") {
             steps {
+                timeout (time: 5, unit: 'MINUTES') {
                     sh "./ci-scripts/createVM.sh --variant cppcheck --job-name ${JOB_NAME} --build-id ${BUILD_ID}"
                 }
             }
+        }
 
         stage ("Start VM -- enb-usrp") {
             steps {
+                timeout (time: 5, unit: 'MINUTES') {
                     sh "./ci-scripts/createVM.sh --variant enb-usrp --job-name ${JOB_NAME} --build-id ${BUILD_ID}"
                 }
             }
+        }
 
         stage ("Start VM -- basic-sim") {
             steps {
+                timeout (time: 5, unit: 'MINUTES') {
                     sh "./ci-scripts/createVM.sh --variant basic-sim --job-name ${JOB_NAME} --build-id ${BUILD_ID}"
                 }
             }
+        }
 
         stage ("Start VM -- phy-sim") {
             steps {
+                timeout (time: 5, unit: 'MINUTES') {
                     sh "./ci-scripts/createVM.sh --variant phy-sim --job-name ${JOB_NAME} --build-id ${BUILD_ID}"
                 }
             }
+        }
 
         stage ("Start VM -- enb-ethernet") {
             steps {
+                timeout (time: 5, unit: 'MINUTES') {
                     sh "./ci-scripts/createVM.sh --variant enb-ethernet --job-name ${JOB_NAME} --build-id ${BUILD_ID}"
                 }
             }
+        }
 
         stage ("Start VM -- ue-ethernet") {
             steps {
+                timeout (time: 5, unit: 'MINUTES') {
                     sh "./ci-scripts/createVM.sh --variant ue-ethernet --job-name ${JOB_NAME} --build-id ${BUILD_ID}"
                 }
             }
+        }
 
         stage ("Variant Builds") {
             parallel {
                 stage ("Analysis with cppcheck") {
                     steps {
                         gitlabCommitStatus(name: "Analysis with cppcheck") {
+                            timeout (time: 20, unit: 'MINUTES') {
                                 sh "./ci-scripts/buildOnVM.sh --workspace $WORKSPACE --variant cppcheck --job-name ${JOB_NAME} --build-id ${BUILD_ID}"
                             }
                         }
                     }
+                }
                 stage ("Build eNb-USRP") {
                     steps {
                         gitlabCommitStatus(name: "Build eNb-USRP") {
+                            timeout (time: 20, unit: 'MINUTES') {
                                 sh "./ci-scripts/buildOnVM.sh --workspace $WORKSPACE --variant enb-usrp --job-name ${JOB_NAME} --build-id ${BUILD_ID}"
                             }
                         }
                     }
+                }
                 stage ("Build basic simulator") {
                     steps {
                         gitlabCommitStatus(name: "Build basic-sim") {
+                            timeout (time: 20, unit: 'MINUTES') {
                                 sh "./ci-scripts/buildOnVM.sh --workspace $WORKSPACE --variant basic-sim --job-name ${JOB_NAME} --build-id ${BUILD_ID} --keep-vm-alive"
                             }
                         }
                     }
+                }
                 stage ("Build physical simulators") {
                     steps {
                         gitlabCommitStatus(name: "Build phy-sim") {
+                            timeout (time: 20, unit: 'MINUTES') {
                                 sh "./ci-scripts/buildOnVM.sh --workspace $WORKSPACE --variant phy-sim --job-name ${JOB_NAME} --build-id ${BUILD_ID} --keep-vm-alive"
                             }
                         }
                     }
+                }
                 stage ("Build eNb-ethernet") {
                     steps {
                         gitlabCommitStatus(name: "Build eNb-ethernet") {
+                            timeout (time: 20, unit: 'MINUTES') {
                                 sh "./ci-scripts/buildOnVM.sh --workspace $WORKSPACE --variant enb-ethernet --job-name ${JOB_NAME} --build-id ${BUILD_ID} --keep-vm-alive"
                             }
                         }
                     }
+                }
                 stage ("Build UE-ethernet") {
                     steps {
                         gitlabCommitStatus(name: "Build UE-ethernet") {
+                            timeout (time: 20, unit: 'MINUTES') {
                                 sh "./ci-scripts/buildOnVM.sh --workspace $WORKSPACE --variant ue-ethernet --job-name ${JOB_NAME} --build-id ${BUILD_ID} --keep-vm-alive"
                             }
                         }
                     }
+                }
                 stage ("Build eNb-USRP on Red Hat") {
                     when {
                         expression {doRedHatBuild}
@@ -203,8 +230,10 @@ pipeline {
                                     withCredentials([
                                         [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.RedHatRemoteCredentials}", usernameVariable: 'RH_Username', passwordVariable: 'RH_Password']
                                     ]) {
+                                        timeout (time: 20, unit: 'MINUTES') {
                                             sh "./ci-scripts/buildOnRH.sh --workspace $WORKSPACE --job-name ${JOB_NAME} --build-id ${BUILD_ID} --remote-host ${params.RedHatRemoteServer} --remote-path ${params.RedHatWorkingPath} --remote-user-name ${RH_Username} --remote-password ${RH_Password}"
                                         }
+                                    }
                                 } catch (Exception e) {
                                     echo "Red Hat build failed not an error now"
                                 }
@@ -237,7 +266,7 @@ pipeline {
                             sh "./ci-scripts/reportBuildLocally.sh --git-url ${GIT_URL} --job-name ${JOB_NAME} --build-id ${BUILD_ID} --trigger push --branch ${GIT_BRANCH} --commit ${GIT_COMMIT}"
                         }
                         if(fileExists('build_results.html')) {
-                            sh "sed -i -e 's#Build-ID: ${BUILD_ID}#Build-ID: <a href=\"${BUILD_URL}\">${BUILD_ID}</a>#' build_results.html"
+                            sh "sed -i -e 's#Build-ID: ${BUILD_ID}#Build-ID: <a href=\"${BUILD_URL}\">${BUILD_ID}</a>#' -e 's#TEMPLATE_BUILD_TIME#${JOB_TIMESTAMP}#' build_results.html"
                             archiveArtifacts artifacts: 'build_results.html'
                         }
                     }
@@ -250,17 +279,21 @@ pipeline {
                 stage ("Test physical simulators") {
                     steps {
                         gitlabCommitStatus(name: "Test phy-sim") {
+                            timeout (time: 20, unit: 'MINUTES') {
                                 sh "./ci-scripts/runTestOnVM.sh --workspace $WORKSPACE --variant phy-sim --job-name ${JOB_NAME} --build-id ${BUILD_ID}"
                             }
                         }
                     }
+                }
                 stage ("Test basic simulator") {
                     steps {
                         gitlabCommitStatus(name: "Test basic-sim") {
+                            timeout (time: 20, unit: 'MINUTES') {
                                 sh "./ci-scripts/runTestOnVM.sh --workspace $WORKSPACE --variant basic-sim --job-name ${JOB_NAME} --build-id ${BUILD_ID} --keep-vm-alive"
                             }
                         }
                     }
+                }
                 stage ("Test on CI bench #1") {
                     stages {
                         stage ("Test FDD - Band 7 - B210") {
@@ -304,6 +337,7 @@ pipeline {
                                                           filter: 'test_results*.html',
                                                           selector: lastCompleted())
                                             if (fileExists('test_results-eNB-CI-FDD-Band7-B210.html')) {
+                                                sh "sed -i -e 's#TEMPLATE_BUILD_TIME#${JOB_TIMESTAMP}#' test_results-eNB-CI-FDD-Band7-B210.html"
                                                 archiveArtifacts artifacts: 'test_results-eNB-CI-FDD-Band7-B210.html'
                                             }
                                         }
@@ -352,6 +386,7 @@ pipeline {
                                                           filter: 'test_results*.html',
                                                           selector: lastCompleted())
                                             if (fileExists('test_results-eNB-CI-TDD-Band40-B210.html')) {
+                                                sh "sed -i -e 's#TEMPLATE_BUILD_TIME#${JOB_TIMESTAMP}#' test_results-eNB-CI-TDD-Band40-B210.html"
                                                 archiveArtifacts artifacts: 'test_results-eNB-CI-TDD-Band40-B210.html'
                                             }
                                         }
@@ -376,7 +411,7 @@ pipeline {
                                 sh "./ci-scripts/reportTestLocally.sh --git-url ${GIT_URL} --job-name ${JOB_NAME} --build-id ${BUILD_ID} --trigger push --branch ${GIT_BRANCH} --commit ${GIT_COMMIT}"
                             }
                             if(fileExists('test_simulator_results.html')) {
-                                sh "sed -i -e 's#Build-ID: ${BUILD_ID}#Build-ID: <a href=\"${BUILD_URL}\">${BUILD_ID}</a>#' test_simulator_results.html"
+                                sh "sed -i -e 's#Build-ID: ${BUILD_ID}#Build-ID: <a href=\"${BUILD_URL}\">${BUILD_ID}</a>#' -e 's#TEMPLATE_BUILD_TIME#${JOB_TIMESTAMP}#' test_simulator_results.html"
                                 archiveArtifacts artifacts: 'test_simulator_results.html'
                             }
                         }

ci-scripts/main.py

@@ -1518,6 +1518,10 @@ class SSHConnection():
 			self.htmlFile.write('  <br>\n')
 			self.htmlFile.write('  <table border = "1">\n')
 			self.htmlFile.write('     <tr>\n')
+			self.htmlFile.write('       <td bgcolor = "lightcyan" >Build Start Time (UTC)</td>\n')
+			self.htmlFile.write('       <td>TEMPLATE_BUILD_TIME</td>\n')
+			self.htmlFile.write('     </tr>\n')
+			self.htmlFile.write('     <tr>\n')
 			self.htmlFile.write('       <td bgcolor = "lightcyan" >GIT Repository</td>\n')
 			self.htmlFile.write('       <td><a href="' + SSH.eNBRepository + '">' + SSH.eNBRepository + '</a></td>\n')
 			self.htmlFile.write('     </tr>\n')

ci-scripts/reportBuildLocally.sh

@@ -438,6 +438,10 @@ echo "  </table>" >> ./build_results.html
 echo "  <br>" >> ./build_results.html
 echo "   <table border = \"1\">" >> ./build_results.html
 echo "      <tr>" >> ./build_results.html
+echo "        <td bgcolor = \"lightcyan\" >Build Start Time (UTC)</td>" >> ./build_results.html
+echo "        <td>TEMPLATE_BUILD_TIME</td>" >> ./build_results.html
+echo "      </tr>" >> ./build_results.html
+echo "      <tr>" >> ./build_results.html
 echo "        <td bgcolor = \"lightcyan\" >GIT Repository</td>" >> ./build_results.html
 echo "        <td>$GIT_URL</td>" >> ./build_results.html
 echo "      </tr>" >> ./build_results.html

ci-scripts/reportTestLocally.sh

@@ -263,6 +263,10 @@ echo "  </table>" >> ./test_simulator_results.html
 echo "  <br>" >> ./test_simulator_results.html
 echo "   <table border = \"1\">" >> ./test_simulator_results.html
 echo "      <tr>" >> ./test_simulator_results.html
+echo "        <td bgcolor = \"lightcyan\" >Build Start Time (UTC)</td>" >> ./test_simulator_results.html
+echo "        <td>TEMPLATE_BUILD_TIME</td>" >> ./test_simulator_results.html
+echo "      </tr>" >> ./test_simulator_results.html
+echo "      <tr>" >> ./test_simulator_results.html
 echo "        <td bgcolor = \"lightcyan\" >GIT Repository</td>" >> ./test_simulator_results.html
 echo "        <td><a href=\"$GIT_URL\">$GIT_URL</a></td>" >> ./test_simulator_results.html
 echo "      </tr>" >> ./test_simulator_results.html