Merge remote-tracking branch 'origin/ci_bring_test_results_to_dashboard' into...

Merge remote-tracking branch 'origin/ci_bring_test_results_to_dashboard' into integration_2021_wk46_c

ci-scripts/Jenkinsfile-git-dashboard

@@ -34,8 +34,11 @@ pipeline {
         stage ("gDashboard") {
             steps {
                 script { 
-                    //retrieve MR data from gitlab and export to gSheet            
-                    sh returnStdout: true, script: 'python3 ci-scripts/ran_dashboard.py'
+                    dir ("ci-scripts/ran_dashboard") {    
+                              //retrieve MR data from gitlab / mySQL db, build HTML pages and load them to AWS S3 bucket (configured as static web page hosting)            
+                              //deprecated method : sh returnStdout: true, script: 'python3 ran_dashboard.py'
+                              sh returnStdout: true, script: 'python3 Hdashboard.py'
+                             }
                 }                                  
             }   
         }

ci-scripts/mysql4testresults/docker-compose.yml

+version: '2'
+services:
+  mysql:
+    container_name: oaicicd_mysql
+    restart: always
+    image: mysql:latest
+    environment:
+      MYSQL_ROOT_PASSWORD: 'ucZBc2XRYdvEm59F' 
+    ports: 
+      - "3307:3306"
+    volumes:
+      - /home/oaicicd/mysql/data:/var/lib/mysql

ci-scripts/mysql4testresults/sql_connect.py

+import pymysql
+import sys
+from datetime import datetime
+
+#This is the script used to write the test results to the mysql DB
+#Called by Jenkins pipeline (jenkinsfile)
+#Must be located in /home/oaicicid/mysql on the database host
+#Usage from Jenkinsfile : 
+#python3 /home/oaicicd/mysql/sql_connect.py ${JOB_NAME} ${params.eNB_MR} ${params.eNB_Branch} ${env.BUILD_ID} ${env.BUILD_URL} ${StatusForDb} ''
+
+class SQLConnect:
+    def __init__(self):
+        self.connection = pymysql.connect(
+                host='172.22.0.2',
+                user='root', 
+                password = 'ucZBc2XRYdvEm59F',
+                db='oaicicd_tests',
+                port=3306
+                )
+
+    def put(self,TEST,MR,BRANCH,BUILD,BUILD_LINK,STATUS):
+        now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        cur=self.connection.cursor()
+        cur.execute ('INSERT INTO test_results (TEST,MR,BRANCH,BUILD,BUILD_LINK,STATUS,DATE) VALUES (%s,%s,%s,%s,%s,%s,%s);' , (TEST, MR, BRANCH, BUILD, BUILD_LINK, STATUS, now))
+        self.connection.commit()
+        self.connection.close()
+
+
+if __name__ == "__main__":
+    mydb=SQLConnect()
+    mydb.put(sys.argv[1],sys.argv[2],sys.argv[3],sys.argv[4],sys.argv[5],sys.argv[6])
+
+

ci-scripts/ran_dashboard/mr_styles.css

+body {
+    font-family: 'lato', sans-serif;
+  }
+
+
+.Main {
+    text-align:left;
+    font-size: 30px;
+    font-weight: bold;   
+}
+
+.DashLink {
+    text-align:left;
+    font-size: 16px;    
+}
+
+.DashLink:hover {
+    font-weight: bold;      
+}
+
+.Date {
+    text-align:left;
+    font-size: 16px;
+    font-weight: bold;      
+}
+
+
+a { text-decoration: none; }
+a:visited { text-decoration: none; color:blue}
+a:hover { text-decoration: none; font-weight: bold; color:blue}
+
+
+.MR_Table {
+    border-collapse: collapse;
+}
+
+.MR_Table tr {
+    font-size: 12px;
+    text-align:center;
+}
+
+
+.MR_Table tr:hover {      
+    background-color:lightgray;
+}
+
+.MR_Table td {
+    border: 1px solid black;
+    padding : 5px;
+}
+.MR_Table td:hover {
+font-weight : bold; 
+}
+
+.MR_Table th {
+    font-size: 14px;
+    text-align:center;
+    padding : 5px;
+}
+
+
+.title_cell {
+    text-align:left;   
+}
+
+
+.MR {
+    table-layout: fixed;
+    width: 50px;
+    background-color: rgb(143, 154, 216);
+}
+
+.CREATED_AT {
+    table-layout: fixed;
+    width: 100px;
+    background-color: rgb(143, 154, 216);
+}
+
+.AUTHOR {
+    table-layout: fixed;
+    width: 150px;
+    background-color: rgb(143, 154, 216);
+}
+
+.TITLE {
+    table-layout: fixed;
+    width: 500px;
+    background-color: rgb(143, 154, 216);
+}
+
+.ASSIGNEE {
+    table-layout: fixed;
+    width: 150px;
+    background-color: rgb(143, 154, 216);
+}
+
+.REVIEWER {
+    table-layout: fixed;
+    width: 150px;
+    background-color: rgb(143, 154, 216);
+}
+
+.CAN_START {
+    background-color: orange;
+    table-layout: fixed;
+    width: 150px;
+}
+.IN_PROGRESS {
+    background-color: yellow;
+    table-layout: fixed;
+    width: 150px;
+}
+.COMPLETED {
+    background-color: rgb(144, 221, 231);
+    table-layout: fixed;
+    width: 150px;
+}
+
+.REVIEW_FORM {
+    table-layout: fixed;
+    width: 100px;
+    background-color: rgb(143, 154, 216);
+}
+
+.OK_MERGE {
+    background-color: rgb(58, 236, 58);
+    table-layout: fixed;
+    width: 150px;
+}
+
+.MERGE_CONFLICTS {
+    table-layout: fixed;
+    width: 100px;
+    background-color: rgb(143, 154, 216);
+}
+

ci-scripts/ran_dashboard/ran_dashboard_cfg.yaml

+LTE-2x2 : #short name used in the dashboard
+  job : 'RAN-LTE-2x2-Module-OAIEPC' #job name from Jenkins, used in the database
+  link : 'https://jenkins-oai.eurecom.fr/view/RAN/job/RAN-LTE-2x2-Module-OAIEPC'
+  bench : 'Obelix-N310-OAIEPC-Quectel(nrmodule2)'
+  test : 'TM2, TDD, 40MHz, MCS9, 26Mb DL, 7Mb UL'
+NSA-B200 :
+  job : 'RAN-NSA-B200-Module-LTEBOX'
+  link : 'https://jenkins-oai.eurecom.fr/view/RAN/job/RAN-NSA-B200-Module-LTEBOX'
+  bench : 'Nepes-B200-Obelix-B200-LTEBOX-Quectel(idefix)'
+  test : 'TM1, 20MHz, 60Mb DL, 3Mb UL'
+NSA-2x2 :
+  job : 'RAN-NSA-2x2-Module-OAIEPC'
+  link : 'https://jenkins-oai.eurecom.fr/view/RAN/job/RAN-NSA-2x2-Module-OAIEPC'
+  bench : 'Asterix-N310-Obelix-N310-OAIEPC-Quectel(nrmodule2)'
+  test : 'TM2, TDD, 40MHz, 60Mb DL, 3Mb UL'
+SA-N310 :
+  job : 'RAN-SA-Module-CN5G'
+  link : 'https://jenkins-oai.eurecom.fr/view/RAN/job/RAN-SA-Module-CN5G'
+  bench : 'Asterix-N310-OAICN5G-Quectel(nrmodule2)'
+  test : 'TM2, TDD, 40MHz, 60Mb DL, 3Mb UL'
+

ci-scripts/ran_dashboard/readme.txt

+The code ran_dashboard.py was initially developped to bring MR status and test resuts to a google sheet, using google sheets API.
+This method is now deprecated, and replaced by Hdashboard.py that builds the results as HTML page and loads it to AWS S3.
+ran_dashboard_cfg.yaml is still used by Hdashboard.py as tests config file.

ci-scripts/ran_dashboard/sqlconnect.py

+#/*
+# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
+# * contributor license agreements.  See the NOTICE file distributed with
+# * this work for additional information regarding copyright ownership.
+# * The OpenAirInterface Software Alliance licenses this file to You under
+# * the OAI Public License, Version 1.1  (the "License"); you may not use this file
+# * except in compliance with the License.
+# * You may obtain a copy of the License at
+# *
+# *      http://www.openairinterface.org/?page_id=698
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# *-------------------------------------------------------------------------------
+# * For more information about the OpenAirInterface (OAI) Software Alliance:
+# *      contact@openairinterface.org
+# */
+#---------------------------------------------------------------------
+# Merge Requests Dashboard for RAN on googleSheet 
+#
+#   Required Python Version
+#     Python 3.x
+#
+#---------------------------------------------------------------------
+
+#author Remi
+
+
+import pymysql
+import sys
+from datetime import datetime
+import pickle
+
+#This is the script/package used by the dashboard to retrieve the MR test results from the database
+
+class SQLConnect:
+    def __init__(self):
+        self.connection = pymysql.connect(
+                host='172.22.0.2',
+                user='root', 
+                password = 'ucZBc2XRYdvEm59F',
+                db='oaicicd_tests',
+                port=3306
+                )
+        self.data={} 
+
+
+    #retrieve data from mysql database and organize it in a dictionary (per MR passed as argument)
+    def get(self,MR):
+        self.data[MR]={}
+        cur=self.connection.cursor()
+
+        #get counters per test
+        sql = "select TEST,STATUS, count(*) AS COUNT from test_results where MR=(%s)  group by TEST, STATUS;"
+        cur.execute(sql,MR)
+        response=cur.fetchall()
+        if len(response)==0:#no test results yet
+            self.data[MR]['PASS']=''
+            self.data[MR]['FAIL']=''
+        else:
+            for i in range(0,len(response)):
+                test=response[i][0]
+                status=response[i][1]
+                count=response[i][2]
+                if test in self.data[MR]:
+                    self.data[MR][test][status]=count
+                else:
+                    self.data[MR][test]={}
+                    self.data[MR][test][status]=count
+      
+        #get last failing build and link
+        sql = "select TEST,BUILD, BUILD_LINK from test_results where MR=(%s) and STATUS='FAIL' order by DATE DESC;"
+        cur.execute(sql,MR)
+        response=cur.fetchall()
+        if len(response)!=0:
+            for i in range(0,len(response)):
+                test=response[i][0]
+                build=response[i][1]
+                link=response[i][2]
+                if 'last_fail' not in self.data[MR][test]:
+                    self.data[MR][test]['last_fail']=[]
+                    self.data[MR][test]['last_fail'].append(build)
+                    self.data[MR][test]['last_fail'].append(link)
+
+        #get last passing build and link
+        sql = "select TEST,BUILD, BUILD_LINK from test_results where MR=(%s) and STATUS='PASS' order by DATE DESC;"
+        cur.execute(sql,MR)
+        response=cur.fetchall()
+        if len(response)!=0:
+            for i in range(0,len(response)):
+                test=response[i][0]
+                build=response[i][1]
+                link=response[i][2]
+                if 'last_pass' not in self.data[MR][test]:
+                    self.data[MR][test]['last_pass']=[]
+                    self.data[MR][test]['last_pass'].append(build)
+                    self.data[MR][test]['last_pass'].append(link)
+
+
+    #close database connection
+    def close_connection(self):
+        self.connection.close()
+
+

ci-scripts/ran_dashboard/test_styles.css

+body {
+    font-family: 'lato', sans-serif;
+  }
+
+.Main {
+    text-align: left;
+    font-size: 30px;
+    font-weight: bold;
+}
+
+
+.DashLink {
+    text-align:left;
+    font-size: 16px;    
+}
+
+.DashLink:hover {
+    font-weight: bold;      
+}
+
+
+.Date {
+    text-align: left;
+    font-size: 16px;
+    font-weight: bold;
+}
+
+h3 {
+    font-size: 16px;    
+}
+
+a { text-decoration: none; }
+a:visited { text-decoration: none; color:blue}
+a:hover { text-decoration: none; font-weight: bold; color:blue}
+
+
+.Test_Table {
+    border-collapse: collapse;
+}
+
+.Test_Table tr {
+    font-size: 12px;
+    text-align:center;
+}
+
+
+.Test_Table tr:hover {
+    background-color:lightgray;
+}
+
+.Test_Table td {
+    border: 1px solid black;
+    padding : 5px;
+}
+.Test_Table td:hover {
+font-weight : bold;
+}
+
+.Test_Table th {
+    font-size: 14px;
+    text-align:center;
+    padding : 5px;
+}
+
+
+
+
+.Test_Name {
+    table-layout: fixed;
+    width: 100px;
+    background-color: rgb(143, 154, 216);
+}
+
+.Test_Descr {
+    table-layout: fixed;
+    width: 400px;
+    background-color: rgb(143, 154, 216);
+}
+
+.Pass {
+    table-layout: fixed;
+    width: 100px;
+    background-color: rgb(143, 154, 216);
+}
+
+.Fail {
+    table-layout: fixed;
+    width: 100px;
+    background-color: rgb(143, 154, 216);
+}
+
+.Last_Pass {
+    table-layout: fixed;
+    width: 100px;
+    background-color: rgb(143, 154, 216);
+}
+
+.Last_Fail {
+    table-layout: fixed;
+    width: 100px;
+    background-color: rgb(143, 154, 216);
+}
+