CI: phytest-timing: detect deviation from normalized values above/below

The CI would only detect a deviation from the normalized value based on
a configurable threshold, typically 1.25 (so we would tolerate a 25%
increase). However, it often happens that a timing gets shorter (e.g.,
forgot to put the timing measurements). This commit introduces a
"Deviation Threshold" (smaller than 1) to not only check that we exceed
1.0 + "Deviation Threshold" (as before), but also that are not below 1.0
- "Deviation Threshold".

In other words, assuming a maximum divergence of 25%, instead of just
checking if val > 1.25, we now also check if val < 0.75 (and flag
error).

ci-scripts/cls_oai_html.py

@@ -437,13 +437,19 @@ class HTMLManagement():
 
 		for k in DataLog['Data']:
 			# TestRow 
+			avg = DataLog['Data'][k][0]
+			maxval = DataLog['Data'][k][1]
+			count = DataLog['Data'][k][2]
+			valnorm = float(DataLog['Data'][k][3])
+			dev = DataLog['DeviationThreshold'][k]
+			ref = DataLog['Ref'][k]
 			self.htmlFile.write('      <tr>\n')
 			self.htmlFile.write('        <td colspan="3" bgcolor = "lightcyan" >' + k  + ' </td>\n')				
-			self.htmlFile.write('        <td colspan="2" bgcolor = "lightcyan" >' + DataLog['Data'][k][0] + '; ' + DataLog['Data'][k][1] + '; ' + DataLog['Data'][k][2] + ' </td>\n')
-			if float(DataLog['Data'][k][3])> DataLog['Threshold'][k]:
-				self.htmlFile.write('        <th bgcolor = "red" >' + DataLog['Data'][k][3]  + ' (Ref = ' + str(DataLog['Ref'][k]) + ' ; Thres = '   +str(DataLog['Threshold'][k])+') ' + '</th>\n')
+			self.htmlFile.write(f'        <td colspan="2" bgcolor = "lightcyan" >{avg}; {maxval}; {count}</td>\n')
+			if valnorm > 1.0 + dev or valnorm < 1.0 - dev:
+				self.htmlFile.write(f'        <th bgcolor = "red" >{valnorm} (Avg over Ref = {avg} over {ref}; max allowed deviation = {dev})</th>\n')
 			else:
-				self.htmlFile.write('        <th bgcolor = "green" ><font color="white">' + DataLog['Data'][k][3]  + ' (Ref = ' + str(DataLog['Ref'][k]) + ' ; Thres = '   +str(DataLog['Threshold'][k])+') ' + '</th>\n')
+				self.htmlFile.write(f'        <th bgcolor = "green" ><font color="white">{valnorm} (Avg over Ref = {avg} over {ref}; max allowed deviation = {dev})</font></th>\n')
 			self.htmlFile.write('      </tr>\n')
 		self.htmlFile.close()
 

ci-scripts/datalog_rt_stats.1x1.60.yaml

@@ -5,7 +5,7 @@ Title : Processing Time (us)
 ColNames :
   - Metric
   - Average; Max; Count
-  - Average vs Reference Deviation (Reference Value; Acceptability Threshold)
+  - Average vs Reference Deviation (Reference Value; Acceptability Deviation Threshold)
 Ref :
   feprx : 46.0
   feptx_prec : 15.0
@@ -20,17 +20,17 @@ Ref :
   DL & UL scheduling timing : 13.0
   UL Indication : 3.0
   Slot Indication : 60.0
-Threshold :
-  feprx : 1.25
-  feptx_prec : 1.25
-  feptx_ofdm : 1.25
-  feptx_total : 1.25
-  L1 Tx processing : 1.25
-  DLSCH encoding : 1.25
-  L1 Rx processing : 1.25
-  PUSCH inner-receiver : 1.25
-  PUSCH decoding : 1.25
-  Schedule Response : 1.25
-  DL & UL scheduling timing : 1.25
-  UL Indication : 1.25
-  Slot Indication : 1.25
+DeviationThreshold :
+  feprx : 0.25
+  feptx_prec : 0.25
+  feptx_ofdm : 0.25
+  feptx_total : 0.25
+  L1 Tx processing : 0.25
+  DLSCH encoding : 0.25
+  L1 Rx processing : 0.25
+  PUSCH inner-receiver : 0.25
+  PUSCH decoding : 0.25
+  Schedule Response : 0.25
+  DL & UL scheduling timing : 0.25
+  UL Indication : 0.25
+  Slot Indication : 0.25

ci-scripts/datalog_rt_stats.2x2.yaml

@@ -5,7 +5,7 @@ Title : Processing Time (us)
 ColNames :
   - Metric
   - Average; Max; Count
-  - Average vs Reference Deviation (Reference Value; Acceptability Threshold)
+  - Average vs Reference Deviation (Reference Value; Acceptability Deviation Threshold)
 Ref : 
   feprx : 120.0
   feptx_prec : 30.0
@@ -16,19 +16,21 @@ Ref :
   L1 Rx processing : 175.0
   PUSCH inner-receiver : 100.0
   PUSCH decoding : 180.0 
+  Schedule Response : 111.0
   DL & UL scheduling timing : 37.0
   UL Indication : 3.0
   Slot Indication : 38.0
-Threshold :
-  feprx : 1.25
-  feptx_prec : 1.25
-  feptx_ofdm : 1.25
-  feptx_total : 1.25
-  L1 Tx processing : 1.25
-  DLSCH encoding : 1.25
-  L1 Rx processing : 1.25
-  PUSCH inner-receiver : 1.25
-  PUSCH decoding : 1.25
-  DL & UL scheduling timing : 1.25
-  UL Indication : 1.25
-  Slot Indication : 1.25
+DeviationThreshold :
+  feprx : 0.25
+  feptx_prec : 0.25
+  feptx_ofdm : 0.25
+  feptx_total : 0.25
+  L1 Tx processing : 0.25
+  DLSCH encoding : 0.25
+  L1 Rx processing : 0.25
+  PUSCH inner-receiver : 0.25
+  PUSCH decoding : 0.25
+  Schedule Response : 0.25
+  DL & UL scheduling timing : 0.25
+  UL Indication : 0.25
+  Slot Indication : 0.25

ci-scripts/datalog_rt_stats.60.2x2.yaml

@@ -5,7 +5,7 @@ Title : Processing Time (us)
 ColNames :
   - Metric
   - Average; Max; Count
-  - Average vs Reference Deviation (Reference Value; Acceptability Threshold)
+  - Average vs Reference Deviation (Reference Value; Acceptability Deviation Threshold)
 Ref : 
   feprx : 84.0
   feptx_prec : 14.0
@@ -20,17 +20,17 @@ Ref :
   DL & UL scheduling timing : 20.0
   UL Indication : 5.0
   Slot Indication : 127.0
-Threshold :
-  feprx : 1.25
-  feptx_prec : 1.25
-  feptx_ofdm : 1.25
-  feptx_total : 1.25
-  L1 Tx processing : 1.25
-  DLSCH encoding : 1.25
-  L1 Rx processing : 1.25
-  PUSCH inner-receiver : 1.25
-  PUSCH decoding : 1.25
-  Schedule Response : 48.0
-  DL & UL scheduling timing : 1.25
-  UL Indication : 1.25
-  Slot Indication : 1.25
+DeviationThreshold :
+  feprx : 0.25
+  feptx_prec : 0.25
+  feptx_ofdm : 0.25
+  feptx_total : 0.25
+  L1 Tx processing : 0.25
+  DLSCH encoding : 0.25
+  L1 Rx processing : 0.25
+  PUSCH inner-receiver : 0.25
+  PUSCH decoding : 0.25
+  Schedule Response : 0.25
+  DL & UL scheduling timing : 0.25
+  UL Indication : 0.25
+  Slot Indication : 0.25

ci-scripts/datalog_rt_stats.default.yaml

@@ -5,7 +5,7 @@ Title : Processing Time (us)
 ColNames :
   - Metric
   - Average; Max; Count
-  - Average vs Reference Deviation (Reference Value; Acceptability Threshold)
+  - Average vs Reference Deviation (Reference Value; Acceptability Deviation Threshold)
 Ref : 
   feprx : 43.0
   feptx_prec : 13.0
@@ -20,17 +20,17 @@ Ref :
   DL & UL scheduling timing : 10.0
   UL Indication : 3.0
   Slot Indication : 40.0
-Threshold :
-  feprx : 1.25
-  feptx_prec : 1.25
-  feptx_ofdm : 1.25
-  feptx_total : 1.25
-  L1 Tx processing : 1.25
-  DLSCH encoding : 1.25
-  L1 Rx processing : 1.25
-  PUSCH inner-receiver : 1.25
-  PUSCH decoding : 1.25
-  Schedule Response : 1.25
-  DL & UL scheduling timing : 1.25
-  UL Indication : 1.25
-  Slot Indication : 1.25
+DeviationThreshold :
+  feprx : 0.25
+  feptx_prec : 0.25
+  feptx_ofdm : 0.25
+  feptx_total : 0.25
+  L1 Tx processing : 0.25
+  DLSCH encoding : 0.25
+  L1 Rx processing : 0.25
+  PUSCH inner-receiver : 0.25
+  PUSCH decoding : 0.25
+  Schedule Response : 0.25
+  DL & UL scheduling timing : 0.25
+  UL Indication : 0.25
+  Slot Indication : 0.25

ci-scripts/ran.py

@@ -1179,8 +1179,10 @@ class RANManagement():
 				self.datalog_rt_stats=datalog_rt_stats
 				#check if there is a fail => will render the test as failed
 				for k in datalog_rt_stats['Data']:
-					if float(datalog_rt_stats['Data'][k][3])> datalog_rt_stats['Threshold'][k]: #condition for fail : avg/ref is greater than the fixed threshold
-						logging.debug('\u001B[1;30;43m datalog_rt_stats metric ' + k + '=' + datalog_rt_stats['Data'][k][3] + ' > threshold ' + str(datalog_rt_stats['Threshold'][k]) + ' \u001B[0m')
+					valnorm = float(datalog_rt_stats['Data'][k][3])
+					dev = datalog_rt_stats['DeviationThreshold'][k]
+					if valnorm > 1.0 + dev or valnorm < 1.0 - dev: # condition for fail : avg/ref deviates by more than "deviation threshold"
+						logging.debug(f'\u001B[1;30;43m normalized datalog_rt_stats metric {k}={valnorm} deviates by more than {dev}\u001B[0m')
 						RealTimeProcessingIssue = True
 			else:
 				statMsg = 'No real time stats found in the log file\n'