dual thread for pdsch

openair1/PHY/CODING/TESTBENCH/ldpctest.c

@@ -103,17 +103,6 @@ uint16_t NB_UE_INST = 1;
 
 short lift_size[51]= {2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,18,20,22,24,26,28,30,32,36,40,44,48,52,56,60,64,72,80,88,96,104,112,120,128,144,160,176,192,208,224,240,256,288,320,352,384};
 
-/*! \file openair1/PHY/CODING/TESTBENCH/ldpctest.c
- * \brief NCTU OpInConnect
- * \author Terngyin Hsu, Sendren Xu, Nungyi Kuo, Kuankai Hsiung, Kaimi Yang (OpInConnect_NCTU)
- * \email tyhsu@cs.nctu.edu.tw
- * \date 13-05-2020
- * \version 2.0
- * \note
- * \warning
- */
-
-
 // ==[START]multi_lpdc_encoder
 struct timespec start_ts, end_ts, start_per_ts, end_per_ts, start_enc_ts[thread_num_max], end_enc_ts[thread_num_max], start_perenc_ts[thread_num_max], end_perenc_ts[thread_num_max];
 multi_ldpc_encoder ldpc_enc[thread_num_max];

openair1/PHY/NR_TRANSPORT/nr_dlsch.c

@@ -29,6 +29,17 @@
 * \note
 * \warning
 */
+
+/*!\file PHY/NR_TRANSPORT/dlsch_decoding.c
+ * \brief Add triggers for dual thread
+ * \author Terngyin, NY, GK, KM (OpInConnect_NCTU)
+ * \email tyhsu@cs.nctu.edu.tw
+ * \date 24-04-2020
+ * \version 1.1
+ * \note
+ * \warning
+ */
+
 //pipeline scrambling and modulation from Ian
 #include "PHY/phy_extern.h"
 #include "PHY/defs_gNB.h"
@@ -43,10 +54,10 @@
 
 void nr_pdsch_codeword_scrambling(uint8_t *in,
                                   uint32_t size,
-                                  uint8_t q,
+                                  uint8_t q,  //use q
                                   uint32_t Nid,
                                   uint32_t n_RNTI,
-                                  uint32_t* out) {
+                                  uint32_t* out) {  //use q => scrambled_output[q]
 
   uint8_t reset, b_idx;
   uint32_t x1, x2, s=0;
@@ -89,7 +100,7 @@ uint8_t nr_generate_pdsch(NR_gNB_DLSCH_t *dlsch,
   NR_DL_gNB_HARQ_t *harq = dlsch->harq_processes[dci_alloc->harq_pid];
   nfapi_nr_dl_config_dlsch_pdu_rel15_t *rel15 = &harq->dlsch_pdu.dlsch_pdu_rel15;
   nfapi_nr_dl_config_pdcch_parameters_rel15_t pdcch_params = dci_alloc->pdcch_params;
-  uint32_t scrambled_output[NR_MAX_NB_CODEWORDS][NR_MAX_PDSCH_ENCODED_LENGTH>>5];
+  uint32_t scrambled_output[NR_MAX_NB_CODEWORDS][NR_MAX_PDSCH_ENCODED_LENGTH>>5]; //NR_MAX_NB_CODEWORDS 2
   int16_t **mod_symbs = (int16_t**)dlsch->mod_symbs;
   int16_t **tx_layers = (int16_t**)dlsch->txdataF;
   int8_t Wf[2], Wt[2], l0, l_prime[2], delta;
@@ -108,9 +119,10 @@ uint8_t nr_generate_pdsch(NR_gNB_DLSCH_t *dlsch,
   /// CRC, coding, interleaving and rate matching
   AssertFatal(harq->pdu!=NULL,"harq->pdu is null\n");
   start_meas(dlsch_encoding_stats);
-  nr_dlsch_encoding(harq->pdu, frame, slot, dlsch, frame_parms);
+  nr_dlsch_encoding(harq->pdu, frame, slot, dlsch, frame_parms);  //the way to encoder
   stop_meas(dlsch_encoding_stats);
-#ifdef DEBUG_DLSCH
+  //printf("rel15->nb_codewords : %d\n", rel15->nb_codewords);
+#ifdef DEBUG_DLSCH  // ==Show original payload & encoded payload ==***
 printf("PDSCH encoding:\nPayload:\n");
 uint32_t encoded_length = nb_symbols*Qm;
 for (int i=0; i<harq->B>>7; i++) {
@@ -127,7 +139,8 @@ for (int i=0; i<encoded_length>>3; i++) {
 printf("\n");
 #endif
 	long sum  = 0;
-#ifdef thread_for_scrambling_modulation
+#if 0
+//#ifdef thread_for_scrambling_modulation //the way to scrambling & modulation
 //	for(int j = 0;j<100;j++){
 		gNB->complete_scrambling_and_modulation = 0;
 		gNB->complete_modulation = 0;
@@ -150,7 +163,7 @@ printf("\n");
 //	}
 //	printf("averge time = %ld\n",sum/100);
 	
-#else
+#elseif 0//original
   /// scrambling
   start_meas(dlsch_scrambling_stats);
   //printf("nb_codewords = %d encoded_length = %d\n",rel15->nb_codewords,encoded_length);
@@ -195,8 +208,41 @@ for (int i=0; i<nb_symbols>>3; i++) {
   printf("\n");
 }
 #endif
-
 #endif
+
+//[START]multi_genetate_pdsch_proc
+struct timespec start_ts, end_ts;
+
+for (int q=0; q<rel15->nb_codewords; q++) // ==Look out!NR_MAX_NB_CODEWORDS is 2!So we can't let q>2 until spec change
+  memset((void*)scrambled_output[q], 0, (encoded_length>>5)*sizeof(uint32_t));
+uint16_t n_RNTI = (pdcch_params.search_space_type == NFAPI_NR_SEARCH_SPACE_TYPE_UE_SPECIFIC) ? ((pdcch_params.scrambling_id==0)?pdcch_params.rnti:0) : 0;
+uint16_t Nid = (pdcch_params.search_space_type == NFAPI_NR_SEARCH_SPACE_TYPE_UE_SPECIFIC) ? pdcch_params.scrambling_id : config->sch_config.physical_cell_id.value;
+printf("================[Scr_Mod]================\n");
+printf(" [Movement]  [No.]  [Round]  [Cost time] \n");
+//Get value
+for (int q=0; q<2; q++){
+  gNB->multi_encoder[q].f = harq->f;
+  gNB->multi_encoder[q].encoded_length = encoded_length;
+  gNB->multi_encoder[q].Nid = Nid;
+  gNB->multi_encoder[q].n_RNTI = n_RNTI;
+  gNB->multi_encoder[q].scrambled_output = scrambled_output[q]; // ==Need to change ==***
+  gNB->multi_encoder[q].Qm = Qm;
+  gNB->multi_encoder[q].mod_symbs = mod_symbs[q]; // ==Need to change ==***
+}
+//Awake threads
+clock_gettime(CLOCK_MONOTONIC, &start_ts);  //timing
+for (int q=0; q<2; q++){
+  pthread_cond_signal(&(gNB->multi_encoder[q].cond_scr_mod));
+}
+//Wait threads
+for (int q=0; q<2; q++){
+  while(gNB->multi_encoder[q].complete_scr_mod!=1);
+}
+clock_gettime(CLOCK_MONOTONIC, &end_ts);  //timing
+//printf("  Movement    No.    Round    Cost time  \n");
+printf("   Total                      %.2f usec\n", (end_ts.tv_nsec - start_ts.tv_nsec) *1.0 / 1000);
+//[END]multi_genetate_pdsch_proc
+
   /// Layer mapping
   nr_layer_mapping(mod_symbs,
                          rel15->nb_layers,

openair1/PHY/NR_TRANSPORT/nr_dlsch_coding.c

@@ -30,6 +30,16 @@
 * \warning
 */
 
+/*!\file PHY/NR_TRANSPORT/dlsch_coding.c
+ * \brief Add triggers for dual thread
+ * \author Terngyin, NY, GK, KM (OpInConnect_NCTU)
+ * \email tyhsu@cs.nctu.edu.tw
+ * \date 24-04-2020
+ * \version 1.1
+ * \note
+ * \warning
+ */
+
 #include "PHY/defs_gNB.h"
 #include "PHY/phy_extern.h"
 #include "PHY/CODING/coding_extern.h"
@@ -44,6 +54,9 @@
 #include "common/utils/LOG/vcd_signal_dumper.h"
 #include "common/utils/LOG/log.h"
 #include <syscall.h>
+#include <time.h>
+
+//multi_ldpc_encoder ldpc_enc[thread_num_max];  //things in ldpc_encoder
 
 //#define DEBUG_DLSCH_CODING
 //#define DEBUG_DLSCH_FREE 1
@@ -273,22 +286,30 @@ void clean_gNB_dlsch(NR_gNB_DLSCH_t *dlsch)
   }
 }
 
-int nr_dlsch_encoding(unsigned char *a,
+int nr_dlsch_encoding(unsigned char *a, //harq->pdu => dlsch->harq_processes[harq_pid]->b
                       int frame,
                       uint8_t slot,
                       NR_gNB_DLSCH_t *dlsch,
                       NR_DL_FRAME_PARMS* frame_parms)
 {
+  /*
   PHY_VARS_gNB *gNB = RC.gNB[0][0];	
   gNB->complete_encode[0] = 0;
 	gNB->complete_encode[1] = 0;
 	gNB->complete_encode[2] = 0;
 	gNB->complete_encode[3] = 0;
+  clock_gettime(CLOCK_MONOTONIC, &start_ts);//timing
 	for (int t = 0; t < 4; t++){
 		pthread_cond_signal(&gNB->thread_encode[t].cond_encode);
 	}
 	while((gNB->complete_encode[0] != 1) || (gNB->complete_encode[1] != 1) || (gNB->complete_encode[2] != 1) || (gNB->complete_encode[3] != 1));
-  /*
+  clock_gettime(CLOCK_MONOTONIC, &end_ts);//timing
+  printf("%.2f usec\n", (end_ts.tv_nsec - start_ts.tv_nsec) *1.0 / 1000);
+  */
+  
+  PHY_VARS_gNB *gNB = RC.gNB[0][0]; // ==check it! ==***
+  
+  struct timespec start_ts, end_ts;
   unsigned int G;
   unsigned int crc=1;
   uint8_t harq_pid = dlsch->harq_ids[frame&2][slot];
@@ -311,14 +332,14 @@ int nr_dlsch_encoding(unsigned char *a,
   uint16_t length_dmrs = 1;
   float Coderate = 0.0;
   uint8_t Nl = 4;
-  */
+  
   /*
   uint8_t *channel_input[MAX_NUM_DLSCH_SEGMENTS]; //unsigned char
   for(j=0;j<MAX_NUM_DLSCH_SEGMENTS;j++) {
     channel_input[j] = (unsigned char *)malloc16(sizeof(unsigned char) * 68*384);
   }
   */
-  /*
+  
   VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_ENB_DLSCH_ENCODING, VCD_FUNCTION_IN);
 
   A = rel15.transport_block_size;
@@ -332,7 +353,7 @@ int nr_dlsch_encoding(unsigned char *a,
 #ifdef DEBUG_DLSCH_CODING
   printf("encoding thinks this is a new packet \n");
 #endif
-*/
+
     /*
     int i;
     printf("dlsch (tx): \n");
@@ -340,7 +361,7 @@ int nr_dlsch_encoding(unsigned char *a,
       printf("%02x.",a[i]);
     printf("\n");
     */
-/*
+
     if (A > 3824) {
       // Add 24-bit crc (polynomial A) to payload
       crc = crc24a(a,A)>>8;
@@ -418,10 +439,71 @@ int nr_dlsch_encoding(unsigned char *a,
       //ldpc_encoder_optim((unsigned char*)dlsch->harq_processes[harq_pid]->c[r],(unsigned char*)&dlsch->harq_processes[harq_pid]->d[r][0],*Zc,Kb,Kr,BG,NULL,NULL,NULL,NULL);
     }
 
-    for(int j=0;j<(dlsch->harq_processes[harq_pid]->C/8+1);j++) {
-      ldpc_encoder_optim_8seg_multi(dlsch->harq_processes[harq_pid]->c,dlsch->harq_processes[harq_pid]->d,*Zc,Kb,Kr,BG,dlsch->harq_processes[harq_pid]->C,j,NULL,NULL,NULL,NULL);
+    //[START]Get value & Awake threads & Wait threads finish
+    // clock_gettime(CLOCK_MONOTONIC, &start_ts);  //timing
+    // for(int i=0;i<2;i++)
+    // for(int j=0;j<(dlsch->harq_processes[harq_pid]->C/8+1);j++) {
+    //   ldpc_encoder_optim_8seg_multi(dlsch->harq_processes[harq_pid]->c,dlsch->harq_processes[harq_pid]->d,*Zc,Kb,Kr,BG,dlsch->harq_processes[harq_pid]->C,j,NULL,NULL,NULL,NULL);
+    // }
+    // clock_gettime(CLOCK_MONOTONIC, &end_ts);  //timing
+    // //printf("  Movement    No.    Round    Cost time  \n");
+    // printf("   Total     Single           %.2f usec\n", (end_ts.tv_nsec - start_ts.tv_nsec) *1.0 / 1000);
+
+    /*get value*/
+    for(int th=0;th<2;th++){
+      gNB->multi_encoder[th].test_input = dlsch->harq_processes[harq_pid]->c;
+      gNB->multi_encoder[th].channel_input_optim = dlsch->harq_processes[harq_pid]->d;
+      gNB->multi_encoder[th].Zc = *Zc;
+      gNB->multi_encoder[th].Kb = Kb;
+      gNB->multi_encoder[th].block_length = Kr;
+      gNB->multi_encoder[th].BG = BG;
+      gNB->multi_encoder[th].n_segments = dlsch->harq_processes[harq_pid]->C;
     }
 
+    /*cpy c to c_test*/
+    // unsigned char bw_scaling =2; // ==Need to change ==***
+    // for(int r=0;r<MAX_NUM_NR_DLSCH_SEGMENTS/bw_scaling;r++){
+    //   dlsch->harq_processes[i]->c[r] = (uint8_t*)malloc16(8448);
+    //   dlsch->harq_processes[i]->d[r] = (uint8_t*)malloc16(68*384); //max size for coded output
+    // }    
+    // for(int th=0;th<2;th++){
+    //   for(int j=0;j<MAX_NUM_NR_DLSCH_SEGMENTS/bw_scaling;j++){  // ==Why can not just be MAX_NUM_NR_DLSCH_SEGMENTS ==???
+    //     gNB->multi_encoder[th].c_test[j]=(uint8_t*)malloc16(8448);//(unsigned char *)malloc16(sizeof(unsigned char) * Kr/8);
+    //     gNB->multi_encoder[th].d_test[j]=(uint8_t*)malloc16(68*384);//(unsigned char *)malloc16(sizeof(unsigned char) * 68*384);
+    //     memcpy(gNB->multi_encoder[th].c_test[j], dlsch->harq_processes[harq_pid]->c[j], 8448);  // ==Check 8448 ==***
+    //   }
+    // }
+    /*Show c_test*/    
+    // printf("c_test :\n");
+    // for (int i=0; i<3; i++){
+    //   printf("%d", dlsch->harq_processes[harq_pid]->c[0][i]);
+    //   printf("/%d\n", gNB->multi_encoder[0].c_test[0][i]);
+    // }
+    // printf("c_test ptr :\n");
+    // for (int i=0; i<3; i++){
+    //   printf("%p", &dlsch->harq_processes[harq_pid]->c[0][i]);
+    //   printf("/%p\n", &gNB->multi_encoder[0].c_test[0][i]);
+    // }
+
+    printf("================[Encoder]================\n");
+    printf(" [Movement]  [No.]  [Round]  [Cost time] \n");
+    
+    clock_gettime(CLOCK_MONOTONIC, &start_ts);  //timing
+    for(int th=0;th<2;th++){
+      pthread_cond_signal(&(gNB->multi_encoder[th].cond));
+    }
+    for(int th = 0;th<2;th++){
+      while(gNB->multi_encoder[th].complete!=1);  // ==check if multi_ldpc_enc done ==
+    }
+    clock_gettime(CLOCK_MONOTONIC, &end_ts);  //timing
+    //printf("  Movement    No.    Round    Cost time  \n");
+    printf("   Total                      %.2f usec\n", (end_ts.tv_nsec - start_ts.tv_nsec) *1.0 / 1000);
+    // for(int th = 0;th<2;th++){
+    //   pthread_mutex_destroy(&gNB->multi_encoder[th].mutex);
+    //   pthread_join(gNB->multi_encoder[th].pthread, NULL);
+    // }
+    //[END]Get value & Awake threads & Wait threads finish
+
 
 #ifdef DEBUG_DLSCH_CODING
       write_output("enc_input0.m","enc_in0",&dlsch->harq_processes[harq_pid]->c[0][0],Kr_bytes,1,4);
@@ -493,6 +575,6 @@ int nr_dlsch_encoding(unsigned char *a,
   }
 
   VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_ENB_DLSCH_ENCODING, VCD_FUNCTION_OUT);
-  */
+
   return 0;
 }

openair1/PHY/defs_gNB.h

@@ -30,6 +30,16 @@
  \warning
 */
 
+/*! \file PHY/defs_gNB.h
+ * \brief Add defines and structure for dual thread
+ * \author Terngyin, NY, GK, KM (OpInConnect_NCTU)
+ * \email tyhsu@cs.nctu.edu.tw
+ * \date 24-04-2020
+ * \version 1.1
+ * \note
+ * \warning
+ */
+
 #ifndef __PHY_DEFS_GNB__H__
 #define __PHY_DEFS_GNB__H__
 
@@ -43,8 +53,41 @@
 #include "PHY/CODING/nrLDPC_decoder/nrLDPC_decoder.h"
 #include "PHY/CODING/nrLDPC_decoder/nrLDPC_types.h"
 
+#include "common/utils/LOG/vcd_signal_dumper.h" //VCD
+
 #define MAX_NUM_RU_PER_gNB MAX_NUM_RU_PER_eNB
 
+typedef struct{
+  /*params of thread*/
+  int id;
+  volatile int flag_wait;
+  pthread_t pthread;
+  pthread_cond_t cond;
+  pthread_cond_t cond_scr_mod;
+  pthread_mutex_t mutex;
+  pthread_mutex_t mutex_scr_mod;
+  pthread_attr_t attr;
+  volatile uint8_t complete;
+  volatile uint8_t complete_scr_mod;
+  /*encoder*/
+  unsigned char **test_input;
+  unsigned char **channel_input_optim;
+  int Zc;
+  int Kb;
+  short block_length;
+  short BG;
+  int n_segments;
+  //unsigned int macro_num; //Not necessary to do
+  /*scrambling & modulation*/
+  uint8_t *f;
+  uint32_t *scrambled_output;
+  int16_t *mod_symbs;
+  uint32_t encoded_length;
+  uint16_t Nid;
+  uint16_t n_RNTI;
+  uint8_t Qm;
+}multi_ldpc_encoder_gNB;
+
 typedef struct {
   uint32_t pbch_a;
   uint32_t pbch_a_interleaved;
@@ -874,7 +917,8 @@ typedef struct PHY_VARS_gNB_s {
   //**************************DLSCH ENCODING**************************//
   dlsch_encoding_ISIP thread_encode[4];
   ldpc_encoding_ISIP ldpc_encode;
-  
+  multi_ldpc_encoder_gNB multi_encoder[2];
+
   volatile uint8_t complete_encode[4];
   
   //**************************DLSCH ENCODING**************************//

openair1/SIMULATION/NR_PHY/dlsim.c

@@ -18,6 +18,8 @@
  * For more information about the OpenAirInterface (OAI) Software Alliance:
  *      contact@openairinterface.org
  */
+
+
 #define _GNU_SOURCE
 #include <sched.h>
 #include <time.h>
@@ -60,6 +62,10 @@
 #include "openair1/SIMULATION/NR_PHY/nr_unitary_defs.h"
 #include "openair1/SIMULATION/NR_PHY/nr_dummy_functions.c"
 #define CPU_AFF
+
+#include "PHY/CODING/nrLDPC_encoder/defs.h"
+struct timespec start_enc_ts[4], end_enc_ts[4];	//timespec
+
 PHY_VARS_gNB *gNB;
 PHY_VARS_NR_UE *UE;
 RAN_CONTEXT_t RC;
@@ -151,7 +157,7 @@ static void *dlsch_encoding_proc(void *ptr){
 	
 	while(!oai_exit){
 		while(pthread_cond_wait(&gNB->thread_encode[num].cond_encode, &gNB->thread_encode[num].mutex_encode) != 0);
-		
+	clock_gettime(CLOCK_MONOTONIC, &start_enc_ts[num]);//timing
 	//	TICK(TIME_DLSCH_ENCODING_THREAD);
 		
 		test->flag_wait = 0;
@@ -275,12 +281,106 @@ static void *dlsch_encoding_proc(void *ptr){
 		}
 		gNB->complete_encode[num] = 1;
 	//	TOCK(TIME_DLSCH_ENCODING_THREAD);
+	clock_gettime(CLOCK_MONOTONIC, &end_enc_ts[num]);//timing
+	printf("%d : %.2f usec\n", num, (end_enc_ts[num].tv_nsec - start_enc_ts[num].tv_nsec) *1.0 / 1000);
 	}
 	encode_status = 0;
 	return &encode_status;
 }
+
+/*! \file openair1/SIMULATION/NR_PHY/dlsim.c
+ * \brief dual thread for pdsch
+ * \author Terngyin Hsu, Sendren Xu, Nungyi Kuo, Kuankai Hsiung, Kaimi Yang (OpInConnect_NCTU)
+ * \email tyhsu@cs.nctu.edu.tw
+ * \date 13-05-2020
+ * \version 2.1
+ * \note
+ * \warning
+ */
   
+//[START]multi_genetate_pdsch_proc
+struct timespec start_encoder_ts[2], end_encoder_ts[2], start_perenc_ts[2], end_perenc_ts[2];
+// int ifRand = 0;
+int vcd = 0;
+static void *multi_genetate_pdsch_proc(void *ptr){
+  //ldpc_encoder_optim_8seg_multi(dlsch->harq_processes[harq_pid]->c,dlsch->harq_processes[harq_pid]->d,*Zc,Kb,Kr,BG,dlsch->harq_processes[harq_pid]->C,j,NULL,NULL,NULL,NULL);
+  //ldpc_encoder_optim_8seg_multi(unsigned char **test_input,unsigned char **channel_input,int Zc,int Kb,short block_length, short BG, int n_segments,unsigned int macro_num, time_stats_t *tinput,time_stats_t *tprep,time_stats_t *tparity,time_stats_t *toutput)
+  multi_ldpc_encoder_gNB *test =(multi_ldpc_encoder_gNB*) ptr;
+  printf("[READY] : %d\n", test->id);
+  while(!oai_exit){
+  	while(pthread_cond_wait(&(gNB->multi_encoder[test->id].cond),&(gNB->multi_encoder[test->id].mutex))!=0);
+  	  if(oai_exit){	//If oai_exit, KILL this thread!
+  	  	pthread_mutex_destroy(&gNB->multi_encoder[test->id].mutex);
+  	  	pthread_mutex_destroy(&gNB->multi_encoder[test->id].mutex_scr_mod);
+      	pthread_join(gNB->multi_encoder[test->id].pthread, NULL);
+      	return 0;
+  	  }
+	  //Print params of multi_ldpc_enc
+	  // printf("[b]ldpc_encoder_optim_8seg: BG %d, Zc %d, Kb %d, block_length %d, segments %d\n",
+	  //       ldpc_enc[test->id].BG,
+	  //       ldpc_enc[test->id].Zc,
+	  //       ldpc_enc[test->id].Kb,
+	  //       ldpc_enc[test->id].block_length,
+	  //       ldpc_enc[test->id].n_segments);
+
+	  int j_start, j_end;
+	  j_start = 0;
+	  j_end = (gNB->multi_encoder[test->id].n_segments/8+1);
+	  int offset = test->id>7?7:test->id;
+	  //printf("[OFFSET] : %d %d\n", offset, test->id);
+	  VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_MULTI_ENC_0 + offset,1);
+	  clock_gettime(CLOCK_MONOTONIC, &start_encoder_ts[test->id]);  //timing
+	  for(int j=j_start;j<j_end;j++){
+	    //printf("  Movement    No.    Round    Cost time  \n");
+	    printf("   Active      %d       %d\n", test->id, j);
+	    clock_gettime(CLOCK_MONOTONIC, &start_perenc_ts[test->id]);  //timing
+	    ldpc_encoder_optim_8seg_multi(gNB->multi_encoder[test->id].test_input,//gNB->multi_encoder[0].c_test,
+	                                  gNB->multi_encoder[test->id].channel_input_optim,//gNB->multi_encoder[0].d_test,
+	                                  gNB->multi_encoder[test->id].Zc,
+	                                  gNB->multi_encoder[test->id].Kb,
+	                                  gNB->multi_encoder[test->id].block_length,
+	                                  gNB->multi_encoder[test->id].BG,
+	                                  gNB->multi_encoder[test->id].n_segments,
+	                                  j,
+	                                  NULL, NULL, NULL, NULL);
+	    clock_gettime(CLOCK_MONOTONIC, &end_perenc_ts[test->id]);  //timing
+	    //printf("  Movement    No.    Round    Cost time  \n");
+	    printf("    Done       %d       %d      %.2f usec\n", test->id, j, (end_perenc_ts[test->id].tv_nsec - start_perenc_ts[test->id].tv_nsec) *1.0 / 1000);
+	  }
+	  clock_gettime(CLOCK_MONOTONIC, &end_encoder_ts[test->id]);  //timing
+	  VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_MULTI_ENC_0 + offset,0);
+	  printf("  All done     %d              %.2f usec\n", test->id, (end_encoder_ts[test->id].tv_nsec - start_encoder_ts[test->id].tv_nsec) *1.0 / 1000);
+	  gNB->multi_encoder[test->id].complete = 1;
+	  //==================================================
+	while(pthread_cond_wait(&(gNB->multi_encoder[test->id].cond_scr_mod),&(gNB->multi_encoder[test->id].mutex_scr_mod))!=0);
+	VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_MULTI_ENC_0 + offset,1);
+	clock_gettime(CLOCK_MONOTONIC, &start_encoder_ts[test->id]);  //timing
+	for (int q=0; q<1; q++){	//Need to change by codewords
+	  printf("   Active      %d       %d\n", test->id, q);
+	  clock_gettime(CLOCK_MONOTONIC, &start_perenc_ts[test->id]);  //timing
+	  nr_pdsch_codeword_scrambling(gNB->multi_encoder[test->id].f,
+	                               gNB->multi_encoder[test->id].encoded_length,
+	                               q,
+	                               gNB->multi_encoder[test->id].Nid,
+	                               gNB->multi_encoder[test->id].n_RNTI,
+	                               gNB->multi_encoder[test->id].scrambled_output);	  
+	  nr_modulation(gNB->multi_encoder[test->id].scrambled_output,
+	                gNB->multi_encoder[test->id].encoded_length,
+	                gNB->multi_encoder[test->id].Qm,
+	                gNB->multi_encoder[test->id].mod_symbs);
+	  clock_gettime(CLOCK_MONOTONIC, &end_perenc_ts[test->id]);  //timing
+	  //printf("  Movement    No.    Round    Cost time  \n");
+	  printf("    Done       %d       %d      %.2f usec\n", test->id, q, (end_perenc_ts[test->id].tv_nsec - start_perenc_ts[test->id].tv_nsec) *1.0 / 1000);
+	}
+	clock_gettime(CLOCK_MONOTONIC, &end_encoder_ts[test->id]);  //timing
+	VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_MULTI_ENC_0 + offset,0);
+	printf("  All done     %d              %.2f usec\n", test->id, (end_encoder_ts[test->id].tv_nsec - start_encoder_ts[test->id].tv_nsec) *1.0 / 1000);
+	gNB->multi_encoder[test->id].complete_scr_mod = 1;
+  }
+  return 0;
+}
 
+//[END]multi_genetate_pdsch_proc
 
 static void *scrambling_proc(void *ptr){
 	
@@ -594,7 +694,7 @@ int main(int argc, char **argv)
 
   randominit(0);
 
-  while ((c = getopt (argc, argv, "f:hA:pf:g:i:j:n:s:S:t:x:y:z:M:N:F:GR:dP:IL:o:a:b:c:j:e:")) != -1) {
+  while ((c = getopt (argc, argv, "f:hA:pf:g:i:j:n:s:S:t:x:y:z:M:N:F:GR:dP:IL:o:a:b:c:j:e:V:")) != -1) {
     switch (c) {
     /*case 'f':
       write_output_file=1;
@@ -783,6 +883,12 @@ int main(int argc, char **argv)
       dlsch_config.mcs_idx = atoi(optarg);
       break;
 
+    case 'V':
+      vcd = atoi(optarg);
+      if(vcd)
+      	T_init(2021, 1, 0);	//VCD init
+      break;
+
     default:
     case 'h':
       printf("%s -h(elp) -p(extended_prefix) -N cell_id -f output_filename -F input_filename -g channel_model -n n_frames -t Delayspread -s snr0 -S snr1 -x transmission_mode -y TXant -z RXant -i Intefrence0 -j Interference1 -A interpolation_file -C(alibration offset dB) -N CellId\n",
@@ -817,7 +923,7 @@ int main(int argc, char **argv)
       exit (-1);
       break;
     }
-  }
+  }  
 
   logInit();
   set_glog(loglvl);
@@ -935,28 +1041,44 @@ int main(int argc, char **argv)
     exit(-1);
   }
   
-  for (int t = 0; t < 4; t++){
-	gNB->thread_encode[t].id = t;
-	gNB->thread_encode[t].flag_wait = 1;
-	pthread_mutex_init( &(gNB->thread_encode[t].mutex_encode), NULL);
-	pthread_cond_init( &(gNB->thread_encode[t].cond_encode), NULL);
-	pthread_create(&(gNB->thread_encode[t].pthread_encode),&gNB->thread_encode[t].attr_encode,dlsch_encoding_proc,&gNB->thread_encode[t]);
-	printf("[CREATE] DLSCH ENCODER Thread %d\n", gNB->thread_encode[t].id);
-  }
-  
-  for(int aa = 0;aa<1;aa++){
-	pthread_attr_init( &gNB->thread_scrambling[aa].attr_scrambling);
-	pthread_mutex_init(&(gNB->thread_scrambling[aa].mutex_tx), NULL);
-	pthread_cond_init(&(gNB->thread_scrambling[aa].cond_tx), NULL);
-	gNB->thread_scrambling[aa].q_id = aa;
-	pthread_create(&(gNB->thread_scrambling[aa].pthread_scrambling),&gNB->thread_scrambling[aa].attr_scrambling,scrambling_proc,&gNB->thread_scrambling[aa]);
-	printf("[CREATE] scrambling_channel thread[%d] \n",aa);
+ //  for (int t = 0; t < 4; t++){	//create threads
+	// gNB->thread_encode[t].id = t;
+	// gNB->thread_encode[t].flag_wait = 1;
+	// pthread_mutex_init( &(gNB->thread_encode[t].mutex_encode), NULL);
+	// pthread_cond_init( &(gNB->thread_encode[t].cond_encode), NULL);
+	// pthread_create(&(gNB->thread_encode[t].pthread_encode),&gNB->thread_encode[t].attr_encode,dlsch_encoding_proc,&gNB->thread_encode[t]);
+	// printf("[CREATE] DLSCH ENCODER Thread %d\n", gNB->thread_encode[t].id);
+ //  }
+
+  //[START]multi_genetate_pdsch_proc:create thread
+  for(int th=0;th<2;th++){
+    pthread_attr_init(&(gNB->multi_encoder[th].attr));
+    pthread_mutex_init(&(gNB->multi_encoder[th].mutex), NULL);
+    pthread_mutex_init(&(gNB->multi_encoder[th].mutex_scr_mod), NULL);
+    pthread_cond_init(&(gNB->multi_encoder[th].cond), NULL);
+    pthread_cond_init(&(gNB->multi_encoder[th].cond_scr_mod), NULL);
+    gNB->multi_encoder[th].id = th;
+    gNB->multi_encoder[th].flag_wait = 1;
+    gNB->multi_encoder[th].complete = 0;
+    gNB->multi_encoder[th].complete_scr_mod = 0;
+    pthread_create(&(gNB->multi_encoder[th].pthread), &(gNB->multi_encoder[th].attr), multi_genetate_pdsch_proc, &(gNB->multi_encoder[th]));    
+    printf("[CREATE] LDPC encoder thread %d \n",gNB->multi_encoder[th].id);
   }
+  //[END]multi_genetate_pdsch_proc:create thread
+
+ //  for(int aa = 0;aa<1;aa++){
+	// pthread_attr_init( &gNB->thread_scrambling[aa].attr_scrambling);
+	// pthread_mutex_init(&(gNB->thread_scrambling[aa].mutex_tx), NULL);
+	// pthread_cond_init(&(gNB->thread_scrambling[aa].cond_tx), NULL);
+	// gNB->thread_scrambling[aa].q_id = aa;
+	// pthread_create(&(gNB->thread_scrambling[aa].pthread_scrambling),&gNB->thread_scrambling[aa].attr_scrambling,scrambling_proc,&gNB->thread_scrambling[aa]);
+	// printf("[CREATE] scrambling_channel thread[%d] \n",aa);
+ //  }
 	
-  pthread_mutex_init( &(gNB->thread_modulation.mutex_tx), NULL);
-  pthread_cond_init( &(gNB->thread_modulation.cond_tx), NULL);
-  pthread_create(&(gNB->thread_modulation.pthread_modulation),&gNB->thread_modulation.attr_modulation,modulation_proc,&gNB->thread_modulation);
-  printf("[CREATE] modulation_channel thread \n");
+  // pthread_mutex_init( &(gNB->thread_modulation.mutex_tx), NULL);
+  // pthread_cond_init( &(gNB->thread_modulation.cond_tx), NULL);
+  // pthread_create(&(gNB->thread_modulation.pthread_modulation),&gNB->thread_modulation.attr_modulation,modulation_proc,&gNB->thread_modulation);
+  // printf("[CREATE] modulation_channel thread \n");
   init_nr_ue_transport(UE,0);
   usleep(1000000);
   nr_gold_pbch(UE);
@@ -1010,7 +1132,7 @@ int main(int argc, char **argv)
     Sched_INFO.TX_req    = &gNB_mac->TX_req[0];
     nr_schedule_response(&Sched_INFO);
 
-    phy_procedures_gNB_TX(gNB,frame,slot,0);
+    phy_procedures_gNB_TX(gNB,frame,slot,0);	// ==the way to downlink ==
     
     //nr_common_signal_procedures (gNB,frame,subframe);
     int txdataF_offset = (slot%2) * frame_parms->samples_per_slot_wCP;
@@ -1304,6 +1426,14 @@ int main(int argc, char **argv)
 
   } // NSR
 
+//[START]Send Kill massage
+  oai_exit = 1;	// ==We should do for threading ==***
+  printf("Kill them all!\n");
+  for(int th=0; th<2; th++){
+  	  pthread_cond_signal(&(gNB->multi_encoder[th].cond));
+	}
+//[END]Send Kill massage
+
   for (i = 0; i < 2; i++) {
     free(s_re[i]);
     free(s_im[i]);