Merge remote-tracking branch 'origin/t2-offload-mr2' into integration_2023_w51

CMakeLists.txt

@@ -500,10 +500,10 @@ target_link_libraries(shlib_loader PRIVATE CONFIG_LIB)
 add_boolean_option(ENABLE_LDPC_T2 OFF "Build support for LDPC Offload to T2 library" OFF)
 set(ENV{PKG_CONFIG_PATH} "$ENV{PKG_CONFIG_PATH}:/usr/local/lib/x86_64-linux-gnu/pkgconfig")
 if (ENABLE_LDPC_T2)
-  pkg_check_modules(LIBDPDK_T2 REQUIRED libdpdk=20.11.7)
-  find_library(PMD_T2 NAMES rte_baseband_accl_ldpc HINTS "/opt/dpdk-t2/lib/x86_64-linux-gnu/")
+  pkg_check_modules(LIBDPDK_T2 REQUIRED libdpdk=20.11.9)
+  find_library(PMD_T2 NAMES rte_baseband_accl_ldpc HINTS ${LIBDPDK_T2_LIBRARY_DIRS})
   if (NOT PMD_T2)
-    message(FATAL_ERROR "could not find PMD T2")
+    message(FATAL_ERROR "could not find poll-mode driver for AccelerComm T2 LDPC Offload (rte_baseband_accl_ldpc.so)")
   endif()
   message(STATUS "T2 build: use ${PMD_T2}")
   add_library(ldpc_t2 MODULE ${OPENAIR1_DIR}/PHY/CODING/nrLDPC_decoder/nrLDPC_decoder_offload.c)

doc/BUILD.md

@@ -184,6 +184,7 @@ Some libraries have further dependencies and might not build on every system:
 - `nrqtscope`: Qt5
 - `ldpc_cuda`: CUDA
 - `websrv`: npm and others
+- `ldpc_t2`: DPDK with patch
 
 # Running `cmake` directly
 

doc/LDPC_T2_OFFLOAD_SETUP.md

@@ -21,19 +21,21 @@ This documentation aims to provide a tutorial for AMD Xilinx T2 Telco card integ
 # Requirements
 
  - bitstream image and PMD driver for the T2 card provided by AccelerComm
- - DPDK 20.11.7 with patch from Accelercomm (also tested with DPDK 20.11.3)
- - tested on RHEL7.9, RHEL9.2, Ubuntu 20.04, Ubuntu 22.04
+ - DPDK 20.11.9 with patch from Accelercomm: version ACCL_BBDEV_DPDK20.11.3_ldpc_3.1.918.patch
+ - tested on RHEL7.9, RHEL9.2, Ubuntu 22.04
 
 # DPDK setup
 ## DPDK installation
+*Note: Following instructions are valid for ACCL_BBDEV_DPDK20.11.3_ldpc_3.1.918.patch version, which is compatible with DPDK 20.11.9. Installation steps, which should be followed for older versions of the patch file (for example ACL_BBDEV_DPDK20.11.3_BL_1006_build_1105_dev_branch_MCT_optimisations_1106_physical_std.patch) are present in older version of this documentation, under the tag 2023.w48.*
+
 ```
 # Get DPDK source code
 git clone https://github.com/DPDK/dpdk-stable.git ~/dpdk-stable
 cd ~/dpdk-stable
-git checkout v20.11.7
-git apply ~/ACL_BBDEV_DPDK20.11.3_xxx.patch
+git checkout v20.11.9
+git apply ~/ACL_BBDEV_DPDK20.11.3_ldpc_3.1.918.patch
 ```
-Replace `~/ACL_BBDEV_DPDK20.11.3_xxx.patch` by patch file provided by
+Replace `~/ACL_BBDEV_DPDK20.11.3_ldpc_3.1.918.patch` by patch file provided by
 Accelercomm.
 ```
 cd ~/dpdk-stable
@@ -47,16 +49,15 @@ sudo ldconfig
 ## DPDK configuration
  - load required kernel module
 ```
-sudo modprobe igb_uio
-sudo insmod ~/dpdk-stable/kernel/linux/igb_uio/igb_uio.ko
+sudo modprobe vfio-pci
 ```
  - check presence of the card and its PCI addres on the host machine
 ```
 lspci | grep "Xilinx"
 ```
- - bind the card with igb_uio driver
+ - bind the card with vfio-pci driver
 ```
-sudo python3 ~/dpdk-stable/usertools/dpdk-devbind.py -b igb_uio 41:00.0
+sudo python3 ~/dpdk-stable/usertools/dpdk-devbind.py --bind=vfio-pci 41:00.0
 ```
 Replace PCI address of the card *41:00.0* by address detected by *lspci | grep "Xilinx"* command
  - hugepages setup (10 x 1GB hugepages)
@@ -68,12 +69,12 @@ sudo python3 ~/dpdk-stable/usertools/dpdk-hugepages.py -p 1G --setup 10G`
 the host machine*
 
 # Modifications in the OAI code
-## PMD path specification
-Path to the PMD for operating the card is specified in `CMakeLists.txt` file in
+## DPDK lib and PMD path specification
+Path to the DPDK lib and Accelercomm PMD for operating the card is specified in the `CMakeLists.txt` file, in
 *LDPC OFFLOAD library* section. Modify following line based on the location of
-the PMD on your system. By deafult, path to the PMD is set to `/opt/dpdk-t2/lib/x86_64-linux-gnu/`.
+`libdpdk.pc` file associated with the target DPDK library on your system. By default, the path is set to `/usr/local/lib/x86_64-linux-gnu/pkgconfig`.
 ```
-find_library(PMD_T2 NAMES rte_baseband_accl_ldpc HINTS "/opt/dpdk-t2/lib/x86_64-linux-gnu/")
+set(ENV{PKG_CONFIG_PATH} "$ENV{PKG_CONFIG_PATH}:/usr/local/lib/x86_64-linux-gnu/pkgconfig")
 ```
 
 ## T2 card DPDK initialization
@@ -106,7 +107,7 @@ cd ~/openairinterface5g/cmake_targets
 cd ~/openairinterface5g
 source oaienv
 cd cmake_targets
-./build_oai -w USRP --ninja --gNB -P -C --build-lib "ldpc_t2"
+./build_oai -w USRP --ninja --gNB -P --build-lib "ldpc_t2" -C
 ```
 Shared object file *libldpc_t2.so* is created during the compilation. This object is conditionally compiled. Selection of the library to compile is done using *--build-lib ldpc_t2*.
 
@@ -131,14 +132,14 @@ sudo ./nr_dlsim -n300 -s30 -R 106 -e 27 -c
 ```
 
 # OTA test
-Offload of the channel encoding and decoding to the AMD Xilinx T2 card is enabled by *--ldpc-offload-enable 1* option.
+Offload of the channel encoding and decoding to the AMD Xilinx T2 card is enabled by *--ldpc-offload-enable* option.
 
 ## Run OAI gNB with USRP B210
 ```
 cd ~/openairinterface5g
 source oaienv
 cd cmake_targets/ran_build/build
-sudo ./nr-softmodem --sa -O ../../../targets/PROJECTS/GENERIC-NR-5GC/CONF/gnb.sa.band78.fr1.106PRB.usrpb210.conf --ldpc-offload-enable 1
+sudo ./nr-softmodem --sa -O ../../../targets/PROJECTS/GENERIC-NR-5GC/CONF/gnb.sa.band78.fr1.106PRB.usrpb210.conf --ldpc-offload-enable
 ```
 
 # Limitations

executables/nr-gnb.c

@@ -593,7 +593,7 @@ void init_eNB_afterRU(void) {
     LOG_I(PHY,"RC.nb_nr_CC[inst:%d]:%p\n", inst, RC.gNB[inst]);
 
     gNB = RC.gNB[inst];
-    gNB->ldpc_offload_flag = ldpc_offload_flag;
+    gNB->ldpc_offload_flag = get_softmodem_params()->ldpc_offload_flag;
     gNB->reorder_thread_disable = get_softmodem_params()->reorder_thread_disable;
 
     phy_init_nr_gNB(gNB);

executables/nr-softmodem-common.h

@@ -98,7 +98,6 @@
 #define CONFIG_HLP_WORKER_CMD    "two option for worker 'WORKER_DISABLE' or 'WORKER_ENABLE'\n"
 #define CONFIG_HLP_USRP_THREAD   "having extra thead for usrp tx\n"
 #define CONFIG_HLP_DISABLNBIOT   "disable nb-iot, even if defined in config\n"
-#define CONFIG_HLP_LDPC_OFFLOAD  "enable LDPC offload to AMD Xilinx T2 card\n"
 #define CONFIG_HLP_USRP_ARGS     "set the arguments to identify USRP (same syntax as in UHD)\n"
 #define CONFIG_HLP_TX_SUBDEV     "set the arguments to select tx_subdev (same syntax as in UHD)\n"
 #define CONFIG_HLP_RX_SUBDEV     "set the arguments to select rx_subdev (same syntax as in UHD)\n"

executables/nr-softmodem.h

@@ -29,7 +29,6 @@
   {"D" ,                    CONFIG_HLP_DLBM_PHYTEST,   0,                .u64ptr=&dlsch_slot_bitmap,          .defintval=0,                     TYPE_UINT64, 0},        \
   {"U" ,                    CONFIG_HLP_ULBM_PHYTEST,   0,                .u64ptr=&ulsch_slot_bitmap,          .defintval=0,                     TYPE_UINT64, 0},        \
   {"usrp-tx-thread-config", CONFIG_HLP_USRP_THREAD,    0,                .iptr=&usrp_tx_thread,               .defstrval=0,                     TYPE_INT,    0},        \
-  {"ldpc-offload-enable",   CONFIG_HLP_LDPC_OFFLOAD,   0,                .iptr=&ldpc_offload_flag,            .defstrval=0,                     TYPE_INT,    0},        \
   {"uecap_file",            CONFIG_HLP_UECAP_FILE,     0,                .strptr=&uecap_file,                 .defstrval="./uecap_ports1.xml",  TYPE_STRING, 0},        \
   {"s" ,                    CONFIG_HLP_SNR,            0,                .dblptr=&snr_dB,                     .defdblval=25,                    TYPE_DOUBLE, 0},        \
 }
@@ -44,7 +43,6 @@ extern uint32_t target_ul_bw;
 extern uint64_t dlsch_slot_bitmap;
 extern uint64_t ulsch_slot_bitmap;
 extern char *uecap_file;
-extern int ldpc_offload_flag;
 
 // In nr-gnb.c
 extern void init_gNB(int single_thread_flag,int wait_for_sync);

executables/softmodem-common.c

@@ -47,7 +47,6 @@ char *parallel_config=NULL;
 char *worker_config=NULL;
 int usrp_tx_thread = 0;
 char *nfapi_str=NULL;
-int ldpc_offload_flag=0;
 uint8_t nfapi_mode=0;
 
 static mapping softmodem_funcs[] = MAPPING_SOFTMODEM_FUNCTIONS;

executables/softmodem-common.h

@@ -109,7 +109,8 @@ extern "C"
 #define CONFIG_HLP_SYNC_REF      "Sync Reference in Sidelink\n"
 #define CONFIG_HLP_NID1          "Set NID1 value in Sidelink\n"
 #define CONFIG_HLP_NID2          "Set NID2 value in Sidelink\n"
-#define CONFIG_HLP_NOITTI "Do not start itti threads, call queue processing in place, inside the caller thread"
+#define CONFIG_HLP_NOITTI        "Do not start itti threads, call queue processing in place, inside the caller thread"
+#define CONFIG_HLP_LDPC_OFFLOAD  "Enable LDPC offload to AMD Xilinx T2 telco card\n"
 
 /*-----------------------------------------------------------------------------------------------------------------------------------------------------*/
 /*                                            command line parameters common to eNodeB and UE                                                          */
@@ -144,6 +145,7 @@ extern "C"
 #define SYNC_REF            softmodem_params.sync_ref
 #define NID1                softmodem_params.nid1
 #define NID2                softmodem_params.nid2
+#define LDPC_OFFLOAD_FLAG   softmodem_params.ldpc_offload_flag
 
 #define REORDER_THREAD_DISABLE    softmodem_params.reorder_thread_disable
 #define DEFAULT_RFCONFIG_FILE    "/usr/local/etc/syriq/ue.band7.tm1.PRB100.NR40.dat";
@@ -191,7 +193,8 @@ extern int usrp_tx_thread;
   {"disable-stats",         CONFIG_HLP_STATS_DISABLE, PARAMFLAG_BOOL, .iptr=&stats_disabled,                  .defintval=0,             TYPE_INT,    0},  \
   {"nid1",                  CONFIG_HLP_NID1,          0,              .iptr=&NID1,                            .defintval=10,            TYPE_INT,    0},  \
   {"nid2",                  CONFIG_HLP_NID2,          0,              .iptr=&NID2,                            .defintval=1,             TYPE_INT,    0},  \
-    {"no-itti-threads", CONFIG_HLP_NOITTI,  PARAMFLAG_BOOL, .iptr=&softmodem_params.no_itti,  .defintval=0,             TYPE_INT,    0},  \
+  {"no-itti-threads",       CONFIG_HLP_NOITTI,        PARAMFLAG_BOOL, .iptr=&softmodem_params.no_itti,        .defintval=0,             TYPE_INT,    0},  \
+  {"ldpc-offload-enable",   CONFIG_HLP_LDPC_OFFLOAD,  PARAMFLAG_BOOL, .iptr=&LDPC_OFFLOAD_FLAG,               .defstrval=0,             TYPE_INT,    0},  \
 }
 // clang-format on
 
@@ -241,6 +244,7 @@ extern int usrp_tx_thread;
     { .s5 = { NULL } },                     \
     { .s5 = { NULL } },                     \
     { .s5 = { NULL } },                     \
+    { .s5 = { NULL } },                     \
 }
 // clang-format on
 
@@ -353,6 +357,7 @@ typedef struct {
   int            nid1;
   int            nid2;
   int no_itti;
+  int ldpc_offload_flag;
 } softmodem_params_t;
 
 extern uint64_t get_softmodem_optmask(void);
@@ -368,7 +373,6 @@ extern int32_t uplink_frequency_offset[MAX_NUM_CCs][4];
 extern int usrp_tx_thread;
 extern uint16_t sl_ahead;
 extern uint16_t sf_ahead;
-extern int ldpc_offload_flag;
 extern int oai_exit;
 
 void tx_func(void *param);

openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_decoder_offload.c

@@ -543,8 +543,8 @@ set_ldpc_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n,
     }
     LOG_D(PHY,"ULSCH %02d HARQPID %02d R %02d COMBIN %d RV %d NCB %05d \n", ulsch_id, harq_pid, r, p_offloadParams->setCombIn, p_offloadParams->rv, p_offloadParams->n_cb);
     ops[i]->ldpc_dec.code_block_mode = 1; // ldpc_dec->code_block_mode;
-    ops[i]->ldpc_dec.harq_combined_input.offset = ulsch_id * (32 * 1024 * 1024) + harq_pid * (2 * 1024 * 1024) + r * (1024 * 32);
-    ops[i]->ldpc_dec.harq_combined_output.offset = ulsch_id * (32 * 1024 * 1024) + harq_pid * (2 * 1024 * 1024) + r * (1024 * 32);
+    ops[i]->ldpc_dec.harq_combined_input.offset = ulsch_id * 64 * LDPC_MAX_CB_SIZE + r * LDPC_MAX_CB_SIZE;
+    ops[i]->ldpc_dec.harq_combined_output.offset = ulsch_id * 64 * LDPC_MAX_CB_SIZE  + r * LDPC_MAX_CB_SIZE;
     if (bufs->hard_outputs != NULL)
       ops[i]->ldpc_dec.hard_output = bufs->hard_outputs[start_idx + i];
     if (bufs->inputs != NULL)
@@ -968,7 +968,7 @@ int32_t LDPCinit()
 
   int socket_id = GET_SOCKET(info.socket_id);
   int out_max_sz = 8448; // max code block size (for BG1), 22 * 384
-  int in_max_sz = 25344; // max number of encoded bits (for BG1), 66 * 384
+  int in_max_sz = LDPC_MAX_CB_SIZE; // max number of encoded bits (for BG2 and MCS0)
   int num_ops = 1;
   int f_ret = create_mempools(ad, socket_id, num_ops, out_max_sz, in_max_sz);
   if (f_ret != TEST_SUCCESS) {

openair1/PHY/CODING/nrLDPC_extern.h

@@ -21,7 +21,8 @@
 #ifndef _NRLDPC_EXTERN_H__
 #define _NRLDPC_EXTERN_H__
 #include "openair1/PHY/CODING/nrLDPC_defs.h"
-
+/* LDPC maximum code block size - maximum E */
+#define LDPC_MAX_CB_SIZE 32768
 /* ldpc coder/decoder API*/
 typedef struct ldpc_interface_s {
   LDPC_initfunc_t *LDPCinit;

openair1/PHY/NR_TRANSPORT/nr_ulsch_decoding.c

@@ -232,8 +232,8 @@ int decode_offload(PHY_VARS_gNB *phy_vars_gNB,
 {
   NR_gNB_ULSCH_t *ulsch = &phy_vars_gNB->ulsch[ULSCH_id];
   NR_UL_gNB_HARQ_t *harq_process = ulsch->harq_process;
-  int16_t z_ol[68 * 384 + 16] __attribute__((aligned(16)));
-  int8_t l_ol[68 * 384 + 16] __attribute__((aligned(16)));
+  int16_t z_ol[LDPC_MAX_CB_SIZE] __attribute__((aligned(16)));
+  int8_t l_ol[LDPC_MAX_CB_SIZE] __attribute__((aligned(16)));
   uint8_t Qm = pusch_pdu->qam_mod_order;
   uint8_t n_layers = pusch_pdu->nrOfLayers;
   const int Kr = harq_process->K;