Merge branch 'nr_cpu_affinity2' into 'develop-nr'

Nr cpu affinity2

See merge request oai/openairinterface5g!478

cmake_targets/build_oai

@@ -339,6 +339,10 @@ function main() {
             FORCE_DEADLINE_SCHEDULER_FLAG_USER="True"
             echo_info "Enabling the usage of deadline scheduler"
             shift 1;;
+       --enable-cpu-affinity)
+            CPU_AFFINITY_FLAG_USER="True"
+            echo_info "Enabling CPU Affinity (only valid when not using deadline scheduler)"
+            shift 1;;
        --disable-cpu-affinity)
             CPU_AFFINITY_FLAG_USER="False"
             echo_info "Disabling CPU Affinity (only valid when not using deadline scheduler)"

targets/ARCH/ADRV9371_ZC706/SYRTEM_IQ_QuickStart.readme

@@ -115,8 +115,16 @@ sudo cset shield --reset
 cd eur_oai_develop-nr/openairinterface5g/
 source oaienv
 
-sudo -E ./cmake_targets/build_oai -c --gNB -w ADRV9371_ZC706 -I
-sudo -E ./cmake_targets/build_oai -c --gNB -w ADRV9371_ZC706
+sudo -E ./cmake_targets/build_oai -c --gNB -w ADRV9371_ZC706 -I --enable-cpu-affinity
+
+/* CPU available shall be mapped on CPU2 to get_nprocs() number with hyperthreading disabled */
+/* because : */
+/* - cset shield leave CPU0 for system */
+/* - CPU1 is reserved for HwRxTx thread for I/Q acquisition ! */
+/* Note: "/usr/local/etc/syriq/cpu-irq.sh" shall be changed if more than 8 cores are used */
+/*   ie. "cset shield --force --kthread on -c 1-7" where "-c 1-7" reserves CPU1 to CPU7 for application (CPU0 is for system) */
+
+sudo -E ./cmake_targets/build_oai -c --gNB -w ADRV9371_ZC706 --enable-cpu-affinity
 
 
 

targets/COMMON/threads_t.h

@@ -2,19 +2,20 @@
 #define _THREADS_T_H_
 
 typedef struct threads_s {
-    int iq;
-    int one;
-    int two;
-    int three;
-    int slot1_proc_one;
-    int slot1_proc_two;
-    int slot1_proc_three;
-    int dlsch_td_one;
-	int dlsch_td_two;
-	int dlsch_td_three;
-	int dlsch_td1_one;
-	int dlsch_td1_two;
-	int dlsch_td1_three;
+  int main;
+  int sync;
+  int one;
+  int two;
+  int three;
+  int slot1_proc_one;
+  int slot1_proc_two;
+  int slot1_proc_three;
+  //int dlsch_td_one;
+  //int dlsch_td_two;
+  //int dlsch_td_three;
+  //int dlsch_td1_one;
+  //int dlsch_td1_two;
+  //int dlsch_td1_three;
 } threads_t;
 
 #endif /* _THREADS_T_H_ */

targets/RT/USER/lte-softmodem.c

@@ -253,7 +253,7 @@ extern char uecap_xer[1024];
 char uecap_xer_in=0;
 
 int oaisim_flag=0;
-threads_t threads= {-1,-1,-1,-1,-1,-1,-1};
+threads_t threads= {-1,-1,-1,-1,-1,-1,-1,-1};
 
 /* see file openair2/LAYER2/MAC/main.c for why abstraction_flag is needed
  * this is very hackish - find a proper solution
@@ -906,14 +906,16 @@ int main( int argc, char **argv )
   char cpu_affinity[1024];
   CPU_ZERO(&cpuset);
 #ifdef CPU_AFFINITY
+  int j;
   if (get_nprocs() > 2) {
-    CPU_SET(0, &cpuset);
+    for (j = 2; j < get_nprocs(); j++)
+      CPU_SET(j, &cpuset);
     s = pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset);
     if (s != 0) {
       perror( "pthread_setaffinity_np");
       exit_fun("Error setting processor affinity");
     }
-    LOG_I(HW, "Setting the affinity of main function to CPU 0, for device library to use CPU 0 only!\n");
+    LOG_I(HW, "Setting the affinity of main function to all CPUs, for device library to use CPU 0 only!\n");
   }
 #endif
   

targets/RT/USER/lte-softmodem.h

@@ -169,7 +169,8 @@
 {"clock",                   CONFIG_HLP_CLK,         0,                      uptr:&clock_source,                 defintval:0,                    TYPE_UINT,      0},                     \
 {"wait-for-sync",           NULL,                   PARAMFLAG_BOOL,         iptr:&wait_for_sync,                defintval:0,                    TYPE_INT,       0},                     \
 {"single-thread-enable",    CONFIG_HLP_NOSNGLT,     PARAMFLAG_BOOL,         iptr:&single_thread_flag,           defintval:0,                    TYPE_INT,       0},                     \
-{"threadIQ",                NULL,                   0,                      iptr:&(threads.iq),                 defintval:1,                    TYPE_INT,       0},                     \
+{"threadMain",                NULL,                   0,                      iptr:&(threads.main),                 defintval:1,                    TYPE_INT,       0},                     \
+{"threadSync",                NULL,                   0,                      iptr:&(threads.sync),                 defintval:1,                    TYPE_INT,       0},                     \
 {"threadOneSubframe",       NULL,                   0,                      iptr:&(threads.one),                defintval:1,                    TYPE_INT,       0},                     \
 {"threadTwoSubframe",       NULL,                   0,                      iptr:&(threads.two),                defintval:1,                    TYPE_INT,       0},                     \
 {"threadThreeSubframe",     NULL,                   0,                      iptr:&(threads.three),              defintval:1,                    TYPE_INT,       0},                     \

targets/RT/USER/lte-ue.c

@@ -456,8 +456,8 @@ static void *UE_thread_synch(void *arg)
 
   cpu_set_t cpuset;
   CPU_ZERO(&cpuset);
-  if ( threads.iq != -1 )
-    CPU_SET(threads.iq, &cpuset);
+  if ( threads.sync != -1 )
+    CPU_SET(threads.sync, &cpuset);
   // this thread priority must be lower that the main acquisition thread
   sprintf(threadname, "sync UE %d\n", UE->Mod_id);
   init_thread(100000, 500000, FIFO_PRIORITY-1, &cpuset, threadname);
@@ -1435,8 +1435,8 @@ void *UE_thread(void *arg) {
 
   cpu_set_t cpuset;
   CPU_ZERO(&cpuset);
-  if ( threads.iq != -1 )
-    CPU_SET(threads.iq, &cpuset);
+  if ( threads.main != -1 )
+    CPU_SET(threads.main, &cpuset);
   init_thread(100000, 500000, FIFO_PRIORITY, &cpuset,
 	      "UHD Threads");
 

targets/RT/USER/lte-uesoftmodem.c

@@ -260,7 +260,7 @@ extern char uecap_xer[1024];
 char uecap_xer_in=0;
 
 int oaisim_flag=0;
-threads_t threads= {-1,-1,-1,-1,-1,-1,-1};
+threads_t threads = {-1,-1,-1,-1,-1,-1,-1};
 
 /* see file openair2/LAYER2/MAC/main.c for why abstraction_flag is needed
  * this is very hackish - find a proper solution
@@ -1014,14 +1014,17 @@ printf("~~~~~~~~~~~~~~~~~~~~successfully get the parallel config[%d], worker con
   char cpu_affinity[1024];
   CPU_ZERO(&cpuset);
 #ifdef CPU_AFFINITY
+  int j;
   if (get_nprocs() > 2) {
-    CPU_SET(0, &cpuset);
+    for (j = 2; j < get_nprocs(); j++)
+      CPU_SET(j, &cpuset);
+    
     s = pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset);
     if (s != 0) {
       perror( "pthread_setaffinity_np");
       exit_fun("Error setting processor affinity");
     }
-    LOG_I(HW, "Setting the affinity of main function to CPU 0, for device library to use CPU 0 only!\n");
+    LOG_I(HW, "Setting the affinity of main function to all CPUs, for device library to use CPU 0 only!\n");
   }
 #endif
   

targets/RT/USER/nr-ru.c

@@ -1499,7 +1499,7 @@ static void* ru_thread( void* param ) {
   int                ret;
   int                subframe =9;
   int                frame    =1023; 
-  char               filename[40];
+  char               filename[40],threadname[40];
   int                print_frame = 2;
   int                i = 0;
 
@@ -1508,7 +1508,8 @@ static void* ru_thread( void* param ) {
 
 
   // set default return value
-  thread_top_init("ru_thread",0,870000,1000000,1000000);
+  sprintf(threadname,"ru_thread %d",ru->idx);
+  thread_top_init(threadname,0,870000,1000000,1000000);
 
   LOG_I(PHY,"Starting RU %d (%s,%s),\n",ru->idx,NB_functions[ru->function],NB_timing[ru->if_timing]);
 

targets/RT/USER/nr-softmodem.c

@@ -247,8 +247,6 @@ double cpuf;
 extern char uecap_xer[1024];
 char uecap_xer_in=0;
 
-threads_t threads= {-1,-1,-1,-1,-1,-1,-1};
-
 /* see file openair2/LAYER2/MAC/main.c for why abstraction_flag is needed
  * this is very hackish - find a proper solution
  */
@@ -1036,14 +1034,19 @@ int main( int argc, char **argv )
   char cpu_affinity[1024];
   CPU_ZERO(&cpuset);
 #ifdef CPU_AFFINITY
+  int j;
   if (get_nprocs() > 2) {
-    CPU_SET(0, &cpuset);
+    // CPU_SET(1, &cpuset);
+    for (j = 2; j < get_nprocs(); j++)
+    {
+      CPU_SET(j, &cpuset);
+    }
     s = pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset);
     if (s != 0) {
       perror( "pthread_setaffinity_np");
       exit_fun("Error setting processor affinity");
     }
-    LOG_I(HW, "Setting the affinity of main function to CPU 0, for device library to use CPU 0 only!\n");
+    LOG_I(HW, "Setting the affinity of main function to all CPUs, for device library to use CPU 0 only!\n");
   }
 #endif
   

targets/RT/USER/nr-softmodem.h

@@ -152,12 +152,6 @@
 {"external-clock",        	 CONFIG_HLP_EXCCLK,	PARAMFLAG_BOOL,   uptr:&clock_source,			defintval:0,			   TYPE_INT,	  0},			   \
 {"wait-for-sync",         	 NULL,  		PARAMFLAG_BOOL,   iptr:&wait_for_sync,  		defintval:0,			   TYPE_INT,	  0},			   \
 {"single-thread-disable", 	 CONFIG_HLP_NOSNGLT,	PARAMFLAG_BOOL,   iptr:&single_thread_flag,		defintval:1,			   TYPE_INT,	  0},			   \
-{"threadIQ",              	 NULL,  		0,		  iptr:&(threads.iq),			defintval:1,			   TYPE_INT,	  0},			   \
-{"threadOneSubframe",     	 NULL,  		0,		  iptr:&(threads.one),  		defintval:1,			   TYPE_INT,	  0},			   \
-{"threadTwoSubframe",    	 NULL,  		0,		  iptr:&(threads.two),  		defintval:1,			   TYPE_INT,	  0},			   \
-{"threadThreeSubframe",    	 NULL,  		0,		  iptr:&(threads.three),  		defintval:1,			   TYPE_INT,	  0},			   \
-{"threadSlot1ProcOne",     	 NULL,  		0,		  iptr:&(threads.slot1_proc_one),      	defintval:1,			   TYPE_INT,	  0},			   \
-{"threadSlot1ProcTwo",    	 NULL,  		0,		  iptr:&(threads.slot1_proc_two),      	defintval:1,			   TYPE_INT,	  0},			   \
 {"A" ,  		  	 CONFIG_HLP_TADV,	0,		  uptr:&timing_advance, 		defintval:0,			   TYPE_UINT,	  0},			   \
 {"C" ,  		  	 CONFIG_HLP_DLF,	0,		  uptr:&(downlink_frequency[0][0]),	defuintval:DEFAULT_DLF,  	   TYPE_UINT,	  0},			   \
 {"a" ,  		  	 CONFIG_HLP_CHOFF,	0,		  iptr:&chain_offset,			defintval:0,			   TYPE_INT,	  0},			   \

targets/RT/USER/nr-ue.c

@@ -341,8 +341,8 @@ static void *UE_thread_synch(void *arg) {
 
     cpu_set_t cpuset;
     CPU_ZERO(&cpuset);
-    if ( threads.iq != -1 )
-        CPU_SET(threads.iq, &cpuset);
+    if ( threads.sync != -1 )
+        CPU_SET(threads.sync, &cpuset);
     // this thread priority must be lower that the main acquisition thread
     sprintf(threadname, "sync UE %d", UE->Mod_id);
     init_thread(100000, 500000, FIFO_PRIORITY-1, &cpuset, threadname);
@@ -833,15 +833,15 @@ void *UE_thread(void *arg) {
 
     cpu_set_t cpuset;
     CPU_ZERO(&cpuset);
-    if ( threads.iq != -1 )
-        CPU_SET(threads.iq, &cpuset);
-    init_thread(100000, 500000, FIFO_PRIORITY, &cpuset,
-                "UHD Threads");
+    if ( threads.main != -1 )
+        CPU_SET(threads.main, &cpuset);
+    sprintf(threadname, "Main UE %d", UE->Mod_id);
+    init_thread(100000, 500000, FIFO_PRIORITY, &cpuset,threadname);
+    
     if ((oaisim_flag == 0) && (UE->mode !=loop_through_memory))
         AssertFatal(0== openair0_device_load(&(UE->rfdevice), &openair0_cfg[0]), "");
     UE->rfdevice.host_type = RAU_HOST;
-    sprintf(threadname, "Main UE %d", UE->Mod_id);
-    pthread_setname_np(pthread_self(), threadname);
+
     init_UE_threads(UE);
 
 #ifdef NAS_UE

targets/RT/USER/nr-uesoftmodem.c

@@ -268,7 +268,8 @@ char uecap_xer[1024],uecap_xer_in=0;
 int oaisim_flag=0;
 int emulate_rf = 0;
 
-threads_t threads= {-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1};
+threads_t threads = {-1,-1,-1,-1,-1,-1,-1,-1};
+int threads_offset = 0;
 
 char* usrp_args=NULL;
 char* usrp_clksrc=NULL;
@@ -1029,8 +1030,25 @@ int main( int argc, char **argv ) {
   //dump_frame_parms(frame_parms[0]);
   
   init_openair0();
-  
 
+  // default threads.main = 2
+  // if there are enough processors, all others on subsequent CPUs
+  if (threads_offset>0) {
+    if (threads_offset+8<=get_nprocs()) {
+      if (threads.main<0) threads.main=threads_offset+1;
+      if (threads.sync<0) threads.sync=threads_offset+2;
+      if (threads.one<0) threads.one=threads_offset+3;
+      if (threads.two<0) threads.two=threads_offset+4;
+      if (threads.three<0) threads.three=threads_offset+5;
+      if (threads.slot1_proc_one<0) threads.slot1_proc_one=threads_offset+6;
+      if (threads.slot1_proc_two<0) threads.slot1_proc_two=threads_offset+7;
+      if (threads.slot1_proc_three<0) threads.slot1_proc_three=threads_offset+8;
+    }
+    else {
+      LOG_E(HW,"Not enough CPUs available (nprocs (=%d) >= threadmain (=%d) + 10)\n",get_nprocs(),threads_offset);
+      exit_fun("Error setting thread affinity\n");
+    }
+  }
 
 #ifndef DEADLINE_SCHEDULER
 
@@ -1041,14 +1059,17 @@ int main( int argc, char **argv ) {
     char cpu_affinity[1024];
     CPU_ZERO(&cpuset);
 #ifdef CPU_AFFINITY
-    if (get_nprocs() > 2) {
-        CPU_SET(0, &cpuset);
+    int j;
+    if (get_nprocs()>=2) {
+      for (j = 2; j < get_nprocs(); j++)
+	CPU_SET(j, &cpuset);
+
         s = pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset);
         if (s != 0) {
             perror( "pthread_setaffinity_np");
             exit_fun("Error setting processor affinity");
         }
-        LOG_I(HW, "Setting the affinity of main function to CPU 0, for device library to use CPU 0 only!\n");
+        LOG_I(HW, "Setting the affinity of main function to all CPUs, for device library to use CPU 0 only!\n");
     }
 #endif
 

targets/RT/USER/nr-uesoftmodem.h

@@ -157,13 +157,16 @@
 {"external-clock",          CONFIG_HLP_EXCCLK,      PARAMFLAG_BOOL,         uptr:&clock_source,                 defintval:0,                    TYPE_INT,       0},                     \
 {"wait-for-sync",           NULL,                   PARAMFLAG_BOOL,         iptr:&wait_for_sync,                defintval:0,                    TYPE_INT,       0},                     \
 {"single-thread-disable",   CONFIG_HLP_NOSNGLT,     PARAMFLAG_BOOL,         iptr:&single_thread_flag,           defintval:1,                    TYPE_INT,       0},                     \
-{"threadIQ",                NULL,                   0,                      iptr:&(threads.iq),                 defintval:1,                    TYPE_INT,       0},                     \
-{"threadOneSubframe",       NULL,                   0,                      iptr:&(threads.one),                defintval:1,                    TYPE_INT,       0},                     \
-{"threadTwoSubframe",       NULL,                   0,                      iptr:&(threads.two),                defintval:1,                    TYPE_INT,       0},                     \
-{"threadThreeSubframe",     NULL,                   0,                      iptr:&(threads.three),              defintval:1,                    TYPE_INT,       0},                     \
-{"threadSlot1ProcOne",      NULL,                   0,                      iptr:&(threads.slot1_proc_one),     defintval:1,                    TYPE_INT,       0},                     \
-{"threadSlot1ProcTwo",      NULL,                   0,                      iptr:&(threads.slot1_proc_two),     defintval:1,                    TYPE_INT,       0},                     \
-{"nr_dlsch-demod-shift",    CONFIG_HLP_DLSHIFT,     0,                      iptr:(int32_t *)&nr_dlsch_demod_shift, defintval:0,                    TYPE_INT,       0},                     \
+{"threadoffset",            NULL,                   0,                      iptr:&(threads_offset),             defintval:0,                    TYPE_INT,       0},                     \
+{"threadMain",              NULL,                   0,                      iptr:&(threads.main),               defintval:-1,                    TYPE_INT,       0},                     \
+{"threadSync",              NULL,                   0,                      iptr:&(threads.sync),               defintval:-1,                    TYPE_INT,       0},                     \
+{"threadOneSubframe",       NULL,                   0,                      iptr:&(threads.one),                defintval:-1,                    TYPE_INT,       0},                     \
+{"threadTwoSubframe",       NULL,                   0,                      iptr:&(threads.two),                defintval:-1,                    TYPE_INT,       0},                     \
+{"threadThreeSubframe",     NULL,                   0,                      iptr:&(threads.three),              defintval:-1,                    TYPE_INT,       0},                     \
+{"threadSlot1ProcOne",      NULL,                   0,                      iptr:&(threads.slot1_proc_one),     defintval:-1,                    TYPE_INT,       0},                     \
+{"threadSlot1ProcTwo",      NULL,                   0,                      iptr:&(threads.slot1_proc_two),     defintval:-1,                    TYPE_INT,       0},                     \
+{"threadSlot1ProcThree",    NULL,                   0,                      iptr:&(threads.slot1_proc_three),   defintval:-1,                    TYPE_INT,       0},                     \
+{"nr-dlsch-demod-shift",    CONFIG_HLP_DLSHIFT,     0,                      iptr:(int32_t *)&nr_dlsch_demod_shift, defintval:0,                     TYPE_INT,       0},                     \
 {"A" ,                      CONFIG_HLP_TADV,        0,                      uptr:&timing_advance,               defintval:0,                    TYPE_UINT,      0},                     \
 {"C" ,                      CONFIG_HLP_DLF,         0,                      uptr:&(downlink_frequency[0][0]),   defuintval:2680000000,          TYPE_UINT,      0},                     \
 {"a" ,                      CONFIG_HLP_CHOFF,       0,                      iptr:&chain_offset,                 defintval:0,                    TYPE_INT,       0},                     \

targets/RT/USER/rt_wrapper.c

@@ -294,20 +294,20 @@ void thread_top_init(char *thread_name,
   char cpu_affinity[1024];
   cpu_set_t cpuset;
 
-  /* Set affinity mask to include CPUs 1 to MAX_CPUS */
+  /* Set affinity mask to include CPUs 2 to MAX_CPUS */
   /* CPU 0 is reserved for UHD threads */
   /* CPU 1 is reserved for all RX_TX threads */
-  /* Enable CPU Affinity only if number of CPUs >2 */
+  /* Enable CPU Affinity only if number of CPUs > 2 */
   CPU_ZERO(&cpuset);
 
 #ifdef CPU_AFFINITY
-  if (get_nprocs() > 2)
+  if (affinity == 0) {
+    LOG_W(HW,"thread_top_init() called with affinity==0, but overruled by #ifdef CPU_AFFINITY\n");
+  }
+  else if (get_nprocs() > 2)
   {
-    if (affinity == 0)
-      CPU_SET(0,&cpuset);
-    else
-      for (j = 1; j < get_nprocs(); j++)
-        CPU_SET(j, &cpuset);
+    for (j = 2; j < get_nprocs(); j++)
+      CPU_SET(j, &cpuset);
     s = pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset);
     if (s != 0)
     {
@@ -315,21 +315,29 @@ void thread_top_init(char *thread_name,
       exit_fun("Error setting processor affinity");
     }
   }
+#else //CPU_AFFINITY
+  if (affinity) {
+    LOG_W(HW,"thread_top_init() called with affinity>0, but overruled by #ifndef CPU_AFFINITY.\n");
+  }
 #endif //CPU_AFFINITY
 
   /* Check the actual affinity mask assigned to the thread */
   s = pthread_getaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset);
-  if (s != 0) {
+  if (s != 0)
+  {
     perror( "pthread_getaffinity_np");
     exit_fun("Error getting processor affinity ");
   }
   memset(cpu_affinity,0,sizeof(cpu_affinity));
   for (j = 0; j < 1024; j++)
-    if (CPU_ISSET(j, &cpuset)) {  
+  {
+    if (CPU_ISSET(j, &cpuset))
+    {  
       char temp[1024];
       sprintf (temp, " CPU_%d", j);
       strcat(cpu_affinity, temp);
     }
+  }
 
   memset(&sparam, 0, sizeof(sparam));
   sparam.sched_priority = sched_get_priority_max(SCHED_FIFO);