rework threads in 5G

cmake_targets/CMakeLists.txt

@@ -754,6 +754,7 @@ include_directories("${OPENAIR_BIN_DIR}")
 
 # Legacy exact order
 
+include_directories("${OPENAIR_DIR}/executables")
 include_directories("${OPENAIR2_DIR}/COMMON")
 include_directories("${OPENAIR2_DIR}/UTIL")
 include_directories("${OPENAIR2_DIR}/UTIL/LOG")
@@ -792,10 +793,7 @@ include_directories("${OPENAIR3_DIR}/S1AP")
 include_directories("${OPENAIR2_DIR}/X2AP")
 include_directories("${OPENAIR3_DIR}/UDP")
 include_directories("${OPENAIR3_DIR}/GTPV1-U")
-include_directories("${OPENAIR_DIR}/targets/COMMON")
 include_directories("${OPENAIR_DIR}/targets/ARCH/COMMON")
-include_directories("${OPENAIR_DIR}/targets/ARCH/EXMIMO/USERSPACE/LIB/")
-include_directories("${OPENAIR_DIR}/targets/ARCH/EXMIMO/DEFS")
 include_directories("${OPENAIR2_DIR}/ENB_APP")
 include_directories("${OPENAIR2_DIR}/GNB_APP")
 include_directories("${OPENAIR2_DIR}/ENB_APP/CONTROL_MODULES/MAC")
@@ -2107,7 +2105,7 @@ if (${XFORMS})
     ${OPENAIR1_DIR}/PHY/TOOLS/nr_phy_scope.c
     )
   set(XFORMS_SOURCE_SOFTMODEM
-    ${OPENAIR_TARGETS}/RT/USER/stats.c
+	  ${OPENAIR_DIR}/executables/stats.c
     )
   set(XFORMS_LIBRARIES "forms")
 endif (${XFORMS})
@@ -2335,12 +2333,10 @@ add_executable(nr-softmodem
   ${nr_rrc_h}
   ${s1ap_h}
 #  ${OPENAIR_BIN_DIR}/messages_xml.h
-  ${OPENAIR_TARGETS}/RT/USER/rt_wrapper.c
-  ${OPENAIR_TARGETS}/RT/USER/nr-gnb.c
-  ${OPENAIR_TARGETS}/RT/USER/nr-ru.c
-  ${OPENAIR_TARGETS}/RT/USER/nr-softmodem.c
+  ${OPENAIR_DIR}/executables/nr-gnb.c
+  ${OPENAIR_DIR}/executables/nr-ru.c
+  ${OPENAIR_DIR}/executables/nr-softmodem.c
   ${OPENAIR1_DIR}/SIMULATION/TOOLS/taus.c
-  ${OPENAIR_TARGETS}/COMMON/create_nr_tasks.c
   ${OPENAIR_TARGETS}/ARCH/COMMON/common_lib.c
   ${OPENAIR1_DIR}/SIMULATION/ETH_TRANSPORT/netlink_init.c
   ${OPENAIR_DIR}/common/utils/utils.c
@@ -2373,13 +2369,10 @@ add_executable(nr-softmodem-nos1
   ${rrc_h}
   ${s1ap_h}
 #  ${OPENAIR_BIN_DIR}/messages_xml.h
-  ${OPENAIR_TARGETS}/RT/USER/rt_wrapper.c
-  ${OPENAIR_TARGETS}/RT/USER/nr-gnb.c
-  ${OPENAIR_TARGETS}/RT/USER/nr-ru.c
-  ${OPENAIR_TARGETS}/RT/USER/nr-softmodem.c
+  ${OPENAIR_DIR}/executables/nr-gnb.c
+  ${OPENAIR_DIR}/executables/nr-ru.c
+  ${OPENAIR_DIR}/executables/nr-softmodem.c
   ${OPENAIR1_DIR}/SIMULATION/TOOLS/taus.c
-  ${OPENAIR_TARGETS}/COMMON/create_tasks.c
-  ${OPENAIR_TARGETS}/COMMON/create_nr_tasks.c
   ${OPENAIR_TARGETS}/ARCH/COMMON/common_lib.c
   ${OPENAIR2_DIR}/RRC/NAS/nas_config.c
   ${OPENAIR2_DIR}/RRC/NAS/rb_config.c
@@ -2448,11 +2441,9 @@ add_executable(nr-uesoftmodem-nos1
   ${rrc_h}
   ${s1ap_h}
 #  ${OPENAIR_BIN_DIR}/messages_xml.h
-  ${OPENAIR_TARGETS}/RT/USER/rt_wrapper.c
   ${OPENAIR_DIR}/executables/nr-ue.c
   ${OPENAIR_DIR}/executables/nr-uesoftmodem.c
   ${OPENAIR1_DIR}/SIMULATION/TOOLS/taus.c
-  #${OPENAIR_TARGETS}/COMMON/create_tasks_ue.c
   ${OPENAIR_TARGETS}/ARCH/COMMON/common_lib.c
   ${OPENAIR1_DIR}/SIMULATION/ETH_TRANSPORT/netlink_init.c
   ${OPENAIR_DIR}/common/utils/utils.c

common/utils/ocp_itti/intertask_interface.cpp

@@ -8,6 +8,7 @@
 
 
 #include <intertask_interface.h>
+#include <common/utils/system.h>
 
 typedef struct timer_elm_s {
   timer_type_t      type;     ///< Timer type
@@ -280,22 +281,8 @@ extern "C" {
 
   int itti_create_task(task_id_t task_id, void *(*start_routine)(void *), void *args_p) {
     task_list_t *t=&tasks[task_id];
-    AssertFatal ( pthread_create (&t->thread, NULL, start_routine, args_p) ==0,
-                  "Thread creation for task %d failed!\n", task_id);
-    pthread_setname_np( t->thread, itti_get_task_name(task_id) );
+    threadCreate (&t->thread, start_routine, args_p, itti_get_task_name(task_id),-1,OAI_PRIORITY_RT);
     LOG_I(TMR,"Created Posix thread %s\n",  itti_get_task_name(task_id) );
-#if 1 // BMC test RT prio
-    {
-      int policy;
-      struct sched_param sparam;
-      memset(&sparam, 0, sizeof(sparam));
-      sparam.sched_priority = sched_get_priority_max(SCHED_FIFO)-10;
-      policy = SCHED_FIFO ; 
-      if (pthread_setschedparam(t->thread, policy, &sparam) != 0) {
-	LOG_E(TMR,"task %s : Failed to set pthread priority\n",  itti_get_task_name(task_id) );
-      }
-    }
-#endif    
     return 0;
   }
 

common/utils/system.c

@@ -28,13 +28,23 @@
  * separate process solves this problem.
  */
 
+#define _GNU_SOURCE
 #include "system.h"
 #include <stdio.h>
 #include <stdlib.h>
 #include <unistd.h>
 #include <pthread.h>
 #include <string.h>
-
+#include <stdint.h>
+#include <sys/mman.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <string.h>
+#include <errno.h>
+#include <pthread.h>
+#include <common/utils/assertions.h>
+#include <common/utils/LOG/log.h>
 #define MAX_COMMAND 4096
 
 static int command_pipe_read;
@@ -50,37 +60,37 @@ static int module_initialized = 0;
 /* util functions                                                   */
 /********************************************************************/
 
-static void lock_system(void)
-{
+static void lock_system(void) {
   if (pthread_mutex_lock(&lock) != 0) {
     printf("pthread_mutex_lock fails\n");
     abort();
   }
 }
 
-static void unlock_system(void)
-{
+static void unlock_system(void) {
   if (pthread_mutex_unlock(&lock) != 0) {
     printf("pthread_mutex_unlock fails\n");
     abort();
   }
 }
 
-static void write_pipe(int p, char *b, int size)
-{
+static void write_pipe(int p, char *b, int size) {
   while (size) {
     int ret = write(p, b, size);
+
     if (ret <= 0) exit(0);
+
     b += ret;
     size -= ret;
   }
 }
 
-static void read_pipe(int p, char *b, int size)
-{
+static void read_pipe(int p, char *b, int size) {
   while (size) {
     int ret = read(p, b, size);
+
     if (ret <= 0) exit(0);
+
     b += ret;
     size -= ret;
   }
@@ -95,14 +105,13 @@ static void read_pipe(int p, char *b, int size)
  * when the main process exits, because then a "read" on the pipe
  * will return 0, in which case "read_pipe" exits.
  */
-static void background_system_process(void)
-{
+static void background_system_process(void) {
   int len;
   int ret;
   char command[MAX_COMMAND+1];
 
   while (1) {
-    read_pipe(command_pipe_read, (char*)&len, sizeof(int));
+    read_pipe(command_pipe_read, (char *)&len, sizeof(int));
     read_pipe(command_pipe_read, command, len);
     ret = system(command);
     write_pipe(result_pipe_write, (char *)&ret, sizeof(int));
@@ -114,8 +123,7 @@ static void background_system_process(void)
 /*     return -1 on error, 0 on success                             */
 /********************************************************************/
 
-int background_system(char *command)
-{
+int background_system(char *command) {
   int res;
   int len;
 
@@ -125,18 +133,22 @@ int background_system(char *command)
   }
 
   len = strlen(command)+1;
+
   if (len > MAX_COMMAND) {
     printf("FATAL: command too long. Increase MAX_COMMAND (%d).\n", MAX_COMMAND);
     printf("command was: '%s'\n", command);
     abort();
   }
+
   /* only one command can run at a time, so let's lock/unlock */
   lock_system();
-  write_pipe(command_pipe_write, (char*)&len, sizeof(int));
+  write_pipe(command_pipe_write, (char *)&len, sizeof(int));
   write_pipe(command_pipe_write, command, len);
-  read_pipe(result_pipe_read, (char*)&res, sizeof(int));
+  read_pipe(result_pipe_read, (char *)&res, sizeof(int));
   unlock_system();
+
   if (res == -1 || !WIFEXITED(res) || WEXITSTATUS(res) != 0) return -1;
+
   return 0;
 }
 
@@ -146,17 +158,16 @@ int background_system(char *command)
 /*     to be called very early by the main processing               */
 /********************************************************************/
 
-void start_background_system(void)
-{
+void start_background_system(void) {
   int p[2];
   pid_t son;
-
   module_initialized = 1;
 
   if (pipe(p) == -1) {
     perror("pipe");
     exit(1);
   }
+
   command_pipe_read  = p[0];
   command_pipe_write = p[1];
 
@@ -164,10 +175,11 @@ void start_background_system(void)
     perror("pipe");
     exit(1);
   }
+
   result_pipe_read  = p[0];
   result_pipe_write = p[1];
-
   son = fork();
+
   if (son == -1) {
     perror("fork");
     exit(1);
@@ -181,6 +193,88 @@ void start_background_system(void)
 
   close(result_pipe_read);
   close(command_pipe_write);
-
   background_system_process();
 }
+
+void thread_top_init(char *thread_name,
+                     int affinity,
+                     uint64_t runtime,
+                     uint64_t deadline,
+                     uint64_t period) {
+#ifdef DEADLINE_SCHEDULER
+  struct sched_attr attr;
+  unsigned int flags = 0;
+  attr.size = sizeof(attr);
+  attr.sched_flags = 0;
+  attr.sched_nice = 0;
+  attr.sched_priority = 0;
+  attr.sched_policy   = SCHED_DEADLINE;
+  attr.sched_runtime  = runtime;
+  attr.sched_deadline = deadline;
+  attr.sched_period   = period;
+  
+  AssertFatal(sched_setattr(0, &attr, flags) == 0, "[SCHED] eNB tx thread: sched_setattr failed\n");
+	      
+#else
+#ifdef CPU_AFFINITY
+  /* Set affinity mask to include CPUs 2 to MAX_CPUS */
+  /* CPU 0 is reserved for UHD threads */  /* CPU 1 is reserved for all RX_TX threads */
+  /* Enable CPU Affinity only if number of CPUs > 2 */
+  cpu_set_t cpuset;
+  CPU_ZERO(&cpuset);
+  if (affinity == 0) {
+    LOG_W(HW,"thread_top_init() called with affinity==0, but overruled by #ifdef CPU_AFFINITY\n");
+  } else if (get_nprocs() > 2) {
+    for (j = 2; j < get_nprocs(); j++)
+      CPU_SET(j, &cpuset);
+    AssertFatal( pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset) == 0, "Error setting processor affinity");
+  }
+#endif //CPU_AFFINITY
+  
+  struct sched_param sparam={0};
+  sparam.sched_priority = OAI_PRIORITY_RT;
+  AssertFatal(pthread_setschedparam(pthread_self(),SCHED_FIFO , &sparam) == 0,"Error setting thread priority");
+  pthread_setname_np(pthread_self(), thread_name);
+#endif 
+  mlockall(MCL_CURRENT | MCL_FUTURE);
+}
+
+void threadTopInit(const char* name, const int affinity, const int priority){
+struct sched_param sparam={0};
+  sparam.sched_priority = priority;
+  AssertFatal(pthread_setschedparam(pthread_self(),SCHED_FIFO , &sparam) == 0,"Error setting thread priority");
+  pthread_setname_np(pthread_self(), name);
+  if (affinity != -1 ) {
+    cpu_set_t cpuset;
+    CPU_ZERO(&cpuset);
+    CPU_SET(affinity, &cpuset);
+    AssertFatal( pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset) == 0, "Error setting processor affinity");
+  }
+}
+
+// Block CPU C-states deep sleep
+void configure_linux(void) {
+  int ret;
+  static int latency_target_fd=-1;
+  uint32_t latency_target_value=10; // in microseconds
+  if (latency_target_fd == -1) {
+    if ( (latency_target_fd = open("/dev/cpu_dma_latency", O_RDWR)) != -1 ) {
+      ret = write(latency_target_fd, &latency_target_value, sizeof(latency_target_value));
+      if (ret == 0) {
+	printf("# error setting cpu_dma_latency to %d!: %s\n", latency_target_value, strerror(errno));
+	close(latency_target_fd);
+	latency_target_fd=-1;
+	return;
+      }
+    }
+  }
+  if (latency_target_fd != -1) 
+    LOG_I(HW,"# /dev/cpu_dma_latency set to %dus\n", latency_target_value);
+  else
+    LOG_E(HW,"Can't set /dev/cpu_dma_latency to %dus\n", latency_target_value);
+
+  // Set CPU frequency to it's maximum
+  if ( 0 != system("for d in /sys/devices/system/cpu/cpu[0-9]*; do cat $d/cpufreq/cpuinfo_max_freq > $d/cpufreq/scaling_min_freq; done"))
+	  LOG_W(HW,"Can't set cpu frequency\n");
+  
+}

common/utils/system.h

@@ -21,6 +21,11 @@
 
 #ifndef _SYSTEM_H_OAI_
 #define _SYSTEM_H_OAI_
+#include <stdint.h>
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 
 /****************************************************
  * send a command to the background process
@@ -36,4 +41,23 @@ int background_system(char *command);
 
 void start_background_system(void);
 
+void set_latency_target(void);
+void configure_linux(void);
+
+void threadCreate(pthread_t* t, void * (*func)(void*), void * param, char* name, int affinity, int priority);
+#define OAI_PRIORITY_RT_LOW sched_get_priority_min(SCHED_FIFO)
+#define OAI_PRIORITY_RT sched_get_priority_max(SCHED_FIFO)-10
+#define OAI_PRIORITY_RT_MAX sched_get_priority_max(SCHED_FIFO)
+
+void thread_top_init(char *thread_name,
+                     int affinity,
+                     uint64_t runtime,
+                     uint64_t deadline,
+                     uint64_t period);
+
+#ifdef __cplusplus
+}
+#endif
+
+
 #endif /* _SYSTEM_H_OAI_ */

common/utils/threadPool/thread-pool.c

@@ -49,23 +49,6 @@ void *one_thread(void *arg) {
   struct  one_thread *myThread=(struct  one_thread *) arg;
   struct  thread_pool *tp=myThread->pool;
 
-  // configure the thread core assignment
-  // TBD: reserve the core for us exclusively
-  if ( myThread->coreID >= 0 &&  myThread->coreID < get_nprocs_conf()) {
-    cpu_set_t cpuset;
-    CPU_ZERO(&cpuset);
-    CPU_SET(myThread->coreID, &cpuset);
-    pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset);
-  }
-
-  //Configure the thread scheduler policy for Linux
-  struct sched_param sparam= {0};
-  sparam.sched_priority = sched_get_priority_max(SCHED_RR);
-  pthread_setschedparam(pthread_self(), SCHED_RR, &sparam);
-  // set the thread name for debugging
-  sprintf(myThread->name,"Tpool_%d",myThread->coreID);
-  pthread_setname_np(pthread_self(), myThread->name );
-
   // Infinite loop to process requests
   do {
     notifiedFIFO_elt_t *elt=pullNotifiedFifoRemember(&tp->incomingFifo, myThread);
@@ -106,10 +89,6 @@ void initTpool(char *params,tpool_t *pool, bool performanceMeas) {
   } else
     pool->traceFd=-1;
 
-  //Configure the thread scheduler policy for Linux
-  struct sched_param sparam= {0};
-  sparam.sched_priority = sched_get_priority_max(SCHED_RR)-1;
-  pthread_setschedparam(pthread_self(), SCHED_RR, &sparam);
   pool->activated=true;
   initNotifiedFIFO(&pool->incomingFifo);
   char *saveptr, * curptr;
@@ -136,7 +115,10 @@ void initTpool(char *params,tpool_t *pool, bool performanceMeas) {
         pool->allthreads->coreID=atoi(curptr);
         pool->allthreads->id=pool->nbThreads;
         pool->allthreads->pool=pool;
-        pthread_create(&pool->allthreads->threadID, NULL, one_thread, (void *)pool->allthreads);
+        //Configure the thread scheduler policy for Linux
+        // set the thread name for debugging
+        sprintf(myThread->name,"Tpool_%d",myThread->coreID);
+        threadCreate(&pool->allthreads->threadID, one_thread, (void *)pool->allthreads, Tpool, myThread->name, myThread->coreID, OAI_PRIORITY_RT);
         pool->nbThreads++;
     }
 

common/utils/threadPool/thread-pool.h

@@ -11,6 +11,7 @@
 #include <sys/syscall.h>
 #include <assertions.h>
 #include <LOG/log.h>
+#include <common/utils/system.h>
 
 #ifdef DEBUG
   #define THREADINIT   PTHREAD_ERRORCHECK_MUTEX_INITIALIZER_NP

executables/nr-uesoftmodem.c

@@ -64,7 +64,6 @@ unsigned short config_frames[4] = {2,9,11,13};
 #endif
 
 #include "intertask_interface.h"
-#include "create_tasks.h"
 
 #include "PHY/INIT/phy_init.h"
 #include "system.h"
@@ -311,11 +310,6 @@ void reset_stats(FL_OBJECT *button, long arg) {
 }
 
 static void *scope_thread(void *arg) {
-  struct sched_param sched_param;
-  sched_param.sched_priority = sched_get_priority_min(SCHED_FIFO)+1;
-  sched_setscheduler(0, SCHED_FIFO,&sched_param);
-  printf("Scope thread has priority %d\n",sched_param.sched_priority);
-  //wait the modem is runnign
   sleep(5);
 
   while (!oai_exit) {
@@ -343,12 +337,7 @@ void init_scope(void) {
     fl_show_form (form_ue[UE_id]->lte_phy_scope_ue, FL_PLACE_HOTSPOT, FL_FULLBORDER, title);
     fl_set_button(form_ue[UE_id]->button_0,0);
     fl_set_object_label(form_ue[UE_id]->button_0, "IA Receiver OFF");
-    ret = pthread_create(&forms_thread, NULL, scope_thread, NULL);
-
-    if (ret == 0)
-      pthread_setname_np( forms_thread, "xforms" );
-
-    printf("Scope thread created, ret=%d\n",ret);
+    threadCreate(&forms_thread, scope_thread, NULL, "scope", -1, OAI_PRIORITY_RT_LOW);
   }
 
 #endif
@@ -763,6 +752,7 @@ int main( int argc, char **argv ) {
   // init UE_PF_PO and mutex lock
   pthread_mutex_init(&ue_pf_po_mutex, NULL);
   memset (&UE_PF_PO[0][0], 0, sizeof(UE_PF_PO_t)*NUMBER_OF_UE_MAX*MAX_NUM_CCs);
+  configure_linux();
   mlockall(MCL_CURRENT | MCL_FUTURE);
   init_scope();
   number_of_cards = 1;

executables/threads_t.h

+#ifndef _THREADS_T_H_
+#define _THREADS_T_H_
+
+typedef struct threads_s {
+  int main;
+  int sync;
+  int one;
+  int two;
+  int three;
+  int slot1_proc_one;
+  int slot1_proc_two;
+  int slot1_proc_three;
+  //int dlsch_td_one;
+  //int dlsch_td_two;
+  //int dlsch_td_three;
+  //int dlsch_td1_one;
+  //int dlsch_td1_two;
+  //int dlsch_td1_three;
+} threads_t;
+
+#endif /* _THREADS_T_H_ */

openair1/PHY/NR_TRANSPORT/nr_dci_tools.c

@@ -64,7 +64,7 @@ void nr_fill_cce_list(NR_gNB_DCI_ALLOC_t* dci_alloc, uint16_t n_shift, uint8_t m
   }
 
   if (pdcch_params->cr_mapping_type == NFAPI_NR_CCE_REG_MAPPING_INTERLEAVED) {
-    AssertFatal((N_reg%(bsize*R))==0, "CCE to REG interleaving: Invalid configuration leading to non integer C (N_reg %us, bsize %d R %d)\n",
+    AssertFatal((N_reg % (bsize*R))==0, "CCE to REG interleaving: Invalid configuration leading to non integer C (N_reg %us, bsize %d R %d)\n",
     N_reg, bsize, R);
     C = N_reg/(bsize*R);
   }