changes in PARALLEL_RU_L!_TRX_SPLIT configuration. Correction of a few bugs...

changes in PARALLEL_RU_L!_TRX_SPLIT configuration. Correction of a few bugs related to passing of variables between threads

openair1/PHY/defs_common.h

@@ -1052,15 +1052,42 @@ static inline void wait_sync(char *thread_name) {
   fflush(stderr);
 }
 
-static inline int wakeup_thread(pthread_mutex_t *mutex,pthread_cond_t *cond,int *instance_cnt,char *name) {
+static inline int wakeup_thread(pthread_mutex_t *mutex,pthread_cond_t *cond,int *instance_cnt,char *name, int sleeptime,int sleep_cnt_max) {
   int rc;
+  int sleep_cnt=0;
+
+  if ((rc = pthread_mutex_lock(mutex)) != 0) {
+    LOG_E(PHY, "wakeup_thread(): error locking mutex for %s (%d %s, %p)\n",
+        name, rc, strerror(rc), (void *)mutex);
+    exit_fun("nothing to add");
+    return(-1);
+  }
+  while (*instance_cnt == 0) {
+    if ((rc = pthread_mutex_unlock(mutex)) != 0) {
+       LOG_E(PHY, "wakeup_thread(): error unlocking mutex for %s (%d %s, %p)\n",
+          name, rc, strerror(rc), (void *)mutex);
+      exit_fun("nothing to add");
+      return(-1);
+    }
+    sleep_cnt++;
+    if (sleep_cnt>sleep_cnt_max) return(-1);
+    usleep(sleeptime);
     if ((rc = pthread_mutex_lock(mutex)) != 0) {
       LOG_E(PHY, "wakeup_thread(): error locking mutex for %s (%d %s, %p)\n",
           name, rc, strerror(rc), (void *)mutex);
       exit_fun("nothing to add");
       return(-1);
     }
+  }
   *instance_cnt = *instance_cnt + 1;
+
+  if ((rc = pthread_mutex_unlock(mutex)) != 0) {
+    LOG_E(PHY, "wakeup_thread(): error locking mutex for %s (%d %s, %p)\n",
+        name, rc, strerror(rc), (void *)mutex);
+    exit_fun("nothing to add");
+    return(-1);
+  }
+
   // the thread can now be woken up
   if (pthread_cond_signal(cond) != 0) {
     LOG_E( PHY, "ERROR pthread_cond_signal\n");

openair1/PHY/defs_eNB.h

@@ -403,7 +403,10 @@ typedef struct RU_t_s{
   RU_proc_t            proc;
   /// stats thread pthread descriptor
   pthread_t            ru_stats_thread;
-
+  /// sleep time in us for delaying L1 wakeup 
+  int wakeup_L1_sleeptime;
+  /// maximum number of sleeps
+  int wakeup_L1_sleep_cnt_max;
 } RU_t;
 
 

targets/RT/USER/lte-enb.c

@@ -133,7 +133,7 @@ void init_eNB(int,int);
 void stop_eNB(int nb_inst);
 
 int wakeup_tx(PHY_VARS_eNB *eNB);
-int wakeup_txfh(L1_rxtx_proc_t *proc,PHY_VARS_eNB *eNB);
+int wakeup_txfh(L1_rxtx_proc_t *proc,int frame_tx,int subframe_tx,uint64_t timestamp_tx,PHY_VARS_eNB *eNB);
 void wakeup_prach_eNB(PHY_VARS_eNB *eNB,RU_t *ru,int frame,int subframe);
 #if (LTE_RRC_VERSION >= MAKE_VERSION(14, 0, 0))
   void wakeup_prach_eNB_br(PHY_VARS_eNB *eNB,RU_t *ru,int frame,int subframe);
@@ -323,6 +323,7 @@ static void *L1_thread_tx(void *param) {
   //wait_sync("tx_thread");
 
   while (!oai_exit) {
+    LOG_D(PHY,"Waiting for TX (IC %d)\n",proc->instance_cnt);
     if (wait_on_condition(&proc->mutex,&proc->cond,&proc->instance_cnt,thread_name)<0) break;
 
     if (oai_exit) break;
@@ -336,10 +337,17 @@ static void *L1_thread_tx(void *param) {
     VCD_SIGNAL_DUMPER_DUMP_VARIABLE_BY_NAME(VCD_SIGNAL_DUMPER_VARIABLES_SUBFRAME_NUMBER_RX1_ENB,proc->subframe_rx);
     VCD_SIGNAL_DUMPER_DUMP_VARIABLE_BY_NAME(VCD_SIGNAL_DUMPER_VARIABLES_FRAME_NUMBER_TX1_ENB,proc->frame_tx);
     VCD_SIGNAL_DUMPER_DUMP_VARIABLE_BY_NAME(VCD_SIGNAL_DUMPER_VARIABLES_FRAME_NUMBER_RX1_ENB,proc->frame_rx);
+    LOG_D(PHY,"L1 TX processing %d.%d\n",proc->frame_tx,proc->subframe_tx);
     phy_procedures_eNB_TX(eNB, proc, 1);
     pthread_mutex_lock( &proc->mutex );
+
+    int subframe_tx = proc->subframe_tx;
+    int frame_tx    = proc->frame_tx;
+    uint64_t timestamp_tx = proc->timestamp_tx;
+
     proc->instance_cnt = -1;
 
+    LOG_D(PHY,"L1 TX signaling done for %d.%d\n",proc->frame_tx,proc->subframe_tx);
     // the thread can now be woken up
     if (pthread_cond_signal(&proc->cond) != 0) {
       LOG_E( PHY, "[eNB] ERROR pthread_cond_signal for eNB TXnp4 thread\n");
@@ -347,7 +355,7 @@ static void *L1_thread_tx(void *param) {
     }
 
     pthread_mutex_unlock( &proc->mutex );
-    wakeup_txfh(proc,eNB);
+    wakeup_txfh(proc,frame_tx,subframe_tx,timestamp_tx,eNB);
   }
 
   return 0;
@@ -387,8 +395,10 @@ static void *L1_thread( void *param ) {
     VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME( VCD_SIGNAL_DUMPER_FUNCTIONS_eNB_PROC_RXTX0+(proc->subframe_rx&1), 0 );
     T(T_ENB_MASTER_TICK, T_INT(0), T_INT(proc->frame_rx), T_INT(proc->subframe_rx));
 
+    LOG_D(PHY,"L1RX waiting for RU RX\n");
     if (wait_on_condition(&proc->mutex,&proc->cond,&proc->instance_cnt,thread_name)<0) break;
 
+    LOG_D(PHY,"L1RX starting in %d.%d\n",proc->frame_rx,proc->subframe_rx);
     VCD_SIGNAL_DUMPER_DUMP_VARIABLE_BY_NAME(VCD_SIGNAL_DUMPER_VARIABLES_CPUID_ENB_THREAD_RXTX,sched_getcpu());
     VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME( VCD_SIGNAL_DUMPER_FUNCTIONS_eNB_PROC_RXTX0+(proc->subframe_rx&1), 1 );
     VCD_SIGNAL_DUMPER_DUMP_VARIABLE_BY_NAME(VCD_SIGNAL_DUMPER_VARIABLES_SUBFRAME_NUMBER_TX0_ENB,proc->subframe_tx);
@@ -402,14 +412,19 @@ static void *L1_thread( void *param ) {
       if (rxtx(eNB,proc,thread_name) < 0) break;
     }
 
+    LOG_D(PHY,"L1 RX %d.%d done\n",proc->frame_rx,proc->subframe_rx);
+
     if(get_thread_parallel_conf() == PARALLEL_RU_L1_SPLIT)              phy_procedures_eNB_TX(eNB, proc, 1);
 
-    if (release_thread(&proc->mutex,&proc->instance_cnt,thread_name)<0) break;
 
     if (NFAPI_MODE!=NFAPI_MODE_VNF) {
       if(get_thread_parallel_conf() == PARALLEL_RU_L1_TRX_SPLIT)      wakeup_tx(eNB);
-      else if(get_thread_parallel_conf() == PARALLEL_RU_L1_SPLIT)     wakeup_txfh(proc,eNB);
+      else if(get_thread_parallel_conf() == PARALLEL_RU_L1_SPLIT)     wakeup_txfh(proc,proc->frame_tx,proc->subframe_tx,proc->timestamp_tx,eNB);
     }
+
+    if (release_thread(&proc->mutex,&proc->instance_cnt,thread_name)<0) break;
+
+
   } // while !oai_exit
 
   VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME( VCD_SIGNAL_DUMPER_FUNCTIONS_eNB_PROC_RXTX0+(proc->subframe_rx&1), 0 );
@@ -442,15 +457,19 @@ void eNB_top(PHY_VARS_eNB *eNB, int frame_rx, int subframe_rx, char *string,RU_t
   }
 }
 
-int wakeup_txfh(L1_rxtx_proc_t *proc,PHY_VARS_eNB *eNB) {
+int wakeup_txfh(L1_rxtx_proc_t *proc,int frame_tx,int subframe_tx,uint64_t timestamp_tx,PHY_VARS_eNB *eNB) {
   RU_t *ru;
   RU_proc_t *ru_proc;
   struct timespec wait;
   wait.tv_sec=0;
   wait.tv_nsec=5000000L;
 
+  LOG_D(PHY,"L1 TX Waking up TX FH %d.%d (IC RU TX %d)\n",frame_tx,subframe_tx,proc->instance_cnt_RUs);
+
+  // grab the information for the RU due to the following wait 
+
   if(wait_on_condition(&proc->mutex_RUs,&proc->cond_RUs,&proc->instance_cnt_RUs,"wakeup_txfh")<0) {
-    LOG_E(PHY,"Frame %d, subframe %d: TX FH not ready\n", proc->frame_tx, proc->subframe_tx);
+    LOG_E(PHY,"Frame %d, subframe %d: TX FH not ready\n", frame_tx, subframe_tx);
     return(-1);
   }
 
@@ -476,10 +495,11 @@ int wakeup_txfh(L1_rxtx_proc_t *proc,PHY_VARS_eNB *eNB) {
     }
 
     ru_proc->instance_cnt_eNBs = 0;
-    ru_proc->timestamp_tx = proc->timestamp_tx;
-    ru_proc->subframe_tx  = proc->subframe_tx;
-    ru_proc->frame_tx     = proc->frame_tx;
+    ru_proc->timestamp_tx = timestamp_tx;
+    ru_proc->subframe_tx  = subframe_tx;
+    ru_proc->frame_tx     = frame_tx;
 
+    LOG_D(PHY,"L1 TX Waking up TX FH (2) %d.%d\n",frame_tx,subframe_tx);
     // the thread can now be woken up
     if (pthread_cond_signal(&ru_proc->cond_eNBs) != 0) {
       LOG_E( PHY, "[eNB] ERROR pthread_cond_signal for eNB TXnp4 thread\n");
@@ -507,10 +527,14 @@ int wakeup_tx(PHY_VARS_eNB *eNB) {
     return(-1);
   }
 
+  LOG_D(PHY,"L1 RX %d.%d Waiting to wake up L1 TX %d.%d (IC L1TX %d)\n",L1_proc->frame_rx,L1_proc->subframe_rx,L1_proc->frame_tx,L1_proc->subframe_tx,L1_proc_tx->instance_cnt);
+
   while(L1_proc_tx->instance_cnt == 0) {
     pthread_cond_wait(&L1_proc_tx->cond,&L1_proc_tx->mutex);
   }
 
+  LOG_D(PHY,"L1 RX Got signal that TX %d.%d is done\n",L1_proc_tx->frame_tx,L1_proc_tx->subframe_tx);
+
   L1_proc_tx->instance_cnt = 0;
   L1_proc_tx->subframe_rx   = L1_proc->subframe_rx;
   L1_proc_tx->frame_rx      = L1_proc->frame_rx;
@@ -519,6 +543,8 @@ int wakeup_tx(PHY_VARS_eNB *eNB) {
   L1_proc_tx->timestamp_tx  = L1_proc->timestamp_tx;
 
   // the thread can now be woken up
+  LOG_D(PHY,"L1 RX Waking up L1 TX %d.%d\n",L1_proc->frame_tx,L1_proc->subframe_tx);
+
   if (pthread_cond_signal(&L1_proc_tx->cond) != 0) {
     LOG_E( PHY, "[eNB] ERROR pthread_cond_signal for eNB TXnp4 thread\n");
     exit_fun( "ERROR pthread_cond_signal" );
@@ -536,6 +562,8 @@ int wakeup_rxtx(PHY_VARS_eNB *eNB,RU_t *ru) {
   LTE_DL_FRAME_PARMS *fp = &eNB->frame_parms;
   int i;
   struct timespec wait;
+  int sleep_cnt=0;
+
   pthread_mutex_lock(&proc->mutex_RU);
 
   for (i=0; i<eNB->num_RU; i++) {
@@ -560,18 +588,31 @@ int wakeup_rxtx(PHY_VARS_eNB *eNB,RU_t *ru) {
   wait.tv_sec=0;
   wait.tv_nsec=5000000L;
 
-  if (L1_proc->instance_cnt == 0) {
+
+  // wake up TX for subframe n+sf_ahead
+  // lock the TX mutex and make sure the thread is ready
+  if (pthread_mutex_timedlock(&L1_proc->mutex,&wait) != 0) {
+    LOG_E( PHY, "[eNB] ERROR pthread_mutex_lock for eNB RXTX thread %d (IC %d)\n", L1_proc->subframe_rx&1,L1_proc->instance_cnt );
+    exit_fun( "error locking mutex_rxtx" );
+    return(-1);
+  }
+
+  LOG_D(PHY,"RU RX Waking up rxtx %d.%d (IC L1RX %d)\n",ru_proc->frame_rx,ru_proc->subframe_rx,L1_proc->instance_cnt);
+
+  while (L1_proc->instance_cnt == 0) {
+    pthread_mutex_unlock( &L1_proc->mutex);
+    if (++sleep_cnt == ru->wakeup_L1_sleep_cnt_max) {
       LOG_E(PHY,"Frame %d, subframe %d: RXTX0 thread busy, dropping\n",L1_proc->frame_rx,L1_proc->subframe_rx);
       return(-1);
     }
+    usleep(ru->wakeup_L1_sleeptime);
 
-  // wake up TX for subframe n+sf_ahead
-  // lock the TX mutex and make sure the thread is ready
     if (pthread_mutex_timedlock(&L1_proc->mutex,&wait) != 0) {
       LOG_E( PHY, "[eNB] ERROR pthread_mutex_lock for eNB RXTX thread %d (IC %d)\n", L1_proc->subframe_rx&1,L1_proc->instance_cnt );
       exit_fun( "error locking mutex_rxtx" );
       return(-1);
     }
+  }
 
   ++L1_proc->instance_cnt;
   // We have just received and processed the common part of a subframe, say n.

targets/RT/USER/lte-ru.c

@@ -759,14 +759,7 @@ void rx_rf(RU_t *ru,int *frame,int *subframe) {
   proc->timestamp_phy_tx = proc->timestamp_rx+((sf_ahead-1)*fp->samples_per_tti);
   proc->subframe_phy_tx  = (proc->subframe_rx+(sf_ahead-1))%10;
   proc->frame_phy_tx     = (proc->subframe_rx>(9-(sf_ahead-1))) ? (proc->frame_rx+1)&1023 : proc->frame_rx;
-#else
-  proc->timestamp_tx = proc->timestamp_rx+(sf_ahead*fp->samples_per_tti);
-  proc->subframe_tx  = (proc->subframe_rx+sf_ahead)%10;
-  proc->frame_tx     = (proc->subframe_rx>(9-sf_ahead)) ? (proc->frame_rx+1)&1023 : proc->frame_rx;
 #endif
-  //proc->timestamp_tx = proc->timestamp_rx+(sf_ahead*fp->samples_per_tti);
-  //proc->subframe_tx  = (proc->subframe_rx+sf_ahead)%10;
-  //proc->frame_tx     = (proc->subframe_rx>(9-sf_ahead)) ? (proc->frame_rx+1)&1023 : proc->frame_rx;
   LOG_D(PHY,"RU %d/%d TS %llu (off %d), frame %d, subframe %d\n",
         ru->idx,
         0,
@@ -1197,7 +1190,7 @@ void do_ru_synch(RU_t *ru) {
 void wakeup_L1s(RU_t *ru) {
   int i;
   PHY_VARS_eNB **eNB_list = ru->eNB_list;
-  LOG_D(PHY,"wakeup_L1s (num %d) for RU %d ru->eNB_top:%p\n",ru->num_eNB,ru->idx, ru->eNB_top);
+  LOG_D(PHY,"wakeup_L1s (num %d) for RU %d (%d.%d)\n",ru->num_eNB,ru->idx, ru->proc.frame_rx,ru->proc.subframe_rx);
 
   if (ru->num_eNB==1 && ru->eNB_top!=0 && get_thread_parallel_conf() == PARALLEL_SINGLE_THREAD) {
     // call eNB function directly
@@ -1484,7 +1477,6 @@ static void *ru_thread_tx( void *param ) {
   //pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset);
   //wait_sync("ru_thread_tx");
   wait_on_condition(&proc->mutex_FH1,&proc->cond_FH1,&proc->instance_cnt_FH1,"ru_thread_tx");
-  printf( "ru_thread_tx ready\n");
 
   while (!oai_exit) {
     VCD_SIGNAL_DUMPER_DUMP_VARIABLE_BY_NAME(VCD_SIGNAL_DUMPER_VARIABLES_CPUID_RU_THREAD_TX,sched_getcpu());
@@ -1495,6 +1487,8 @@ static void *ru_thread_tx( void *param ) {
     // wait until eNBs are finished subframe RX n and TX n+4
     wait_on_condition(&proc->mutex_eNBs,&proc->cond_eNBs,&proc->instance_cnt_eNBs,"ru_thread_tx");
 
+    LOG_D(PHY,"ru_thread_tx: TX in %d.%d\n",ru->proc.frame_tx,ru->proc.subframe_tx);
+
     if (oai_exit) break;
 
     // do TX front-end processing if needed (precoding and/or IDFTs)
@@ -1509,7 +1503,7 @@ static void *ru_thread_tx( void *param ) {
 
       if (ru->fh_north_out) ru->fh_north_out(ru);
     }
-
+    LOG_D(PHY,"ru_thread_tx: releasing RU TX in %d.%d\n",proc->frame_tx,proc->subframe_tx);
     release_thread(&proc->mutex_eNBs,&proc->instance_cnt_eNBs,"ru_thread_tx");
 
     for(int i = 0; i<ru->num_eNB; i++) {
@@ -1536,6 +1530,7 @@ static void *ru_thread_tx( void *param ) {
         pthread_mutex_lock( &L1_proc->mutex_RUs);
         L1_proc->instance_cnt_RUs = 0;
 
+        LOG_D(PHY,"ru_thread_tx: Signaling RU TX done in %d.%d\n",proc->frame_tx,proc->subframe_tx);
         // the thread can now be woken up
         if (pthread_cond_signal(&L1_proc->cond_RUs) != 0) {
           LOG_E( PHY, "[eNB] ERROR pthread_cond_signal for eNB TXnp4 thread\n");
@@ -1943,7 +1938,7 @@ static void *eNB_thread_phy_tx( void *param ) {
 
     LOG_D(PHY,"Running eNB phy tx procedures\n");
 
-    if(ru->num_eNB == 1) {
+    AssertFatal(ru->num_eNB == 1, "handle multiple L1 case\n");
     L1_proc.subframe_tx = proc->subframe_phy_tx;
     L1_proc.frame_tx = proc->frame_phy_tx;
     phy_procedures_eNB_TX(eNB_list[0], &L1_proc, 1);
@@ -1967,7 +1962,6 @@ static void *eNB_thread_phy_tx( void *param ) {
     }
 
     pthread_mutex_unlock( &ru->proc.mutex_rf_tx );
-    }
 
     if (release_thread(&proc->mutex_phy_tx,&proc->instance_cnt_phy_tx,"eNB_thread_phy_tx") < 0) break;
 
@@ -2668,6 +2662,8 @@ void init_RU(char *rf_config_file) {
     // use eNB_list[0] as a reference for RU frame parameters
     // NOTE: multiple CC_id are not handled here yet!
 
+    ru->wakeup_L1_sleeptime = 2000;
+    ru->wakeup_L1_sleep_cnt_max  = 3;
     if (ru->num_eNB > 0) {
       LOG_D(PHY, "%s() RC.ru[%d].num_eNB:%d ru->eNB_list[0]:%p RC.eNB[0][0]:%p rf_config_file:%s\n", __FUNCTION__, ru_id, ru->num_eNB, ru->eNB_list[0], RC.eNB[0][0], ru->rf_config_file);
 

targets/RT/USER/lte-ue.c

@@ -890,7 +890,7 @@ void ue_stub_rx_handler(unsigned int num_bytes, char *rx_buffer) {
   switch (((UE_tport_header_t *)rx_buffer)->packet_type) {
     case TTI_SYNC:
       emulator_absSF = ((UE_tport_header_t *)rx_buffer)->absSF;
-      wakeup_thread(&UE->timer_mutex,&UE->timer_cond,&UE->instance_cnt_timer,"timer_thread");
+      wakeup_thread(&UE->timer_mutex,&UE->timer_cond,&UE->instance_cnt_timer,"timer_thread",100,1);
       break;
 
     case SLSCH:
@@ -1736,7 +1736,7 @@ void *UE_thread(void *arg) {
               LOG_E(PHY,"Missed real time\n");
               continue;
             } else {
-              LOG_E(PHY,"System error\n");
+              LOG_E(PHY,"System error %s\n",strerror(errno));
               abort();
             }
           }