Commit 09e32b28 authored by Rahul Gottipati's avatar Rahul Gottipati

Code cleanup and documentation update

parent 78f3bca1
......@@ -60,7 +60,7 @@ void handle_nr_rach(NR_UL_IND_t *UL_info) {
## P7 UL reception at VNF
Through the infinite loop [while(vnf_p7->terminate == 0)] running in nfapi_nr_vnf_p7_start(), the VNF receives and unpacks the uplink indication message received on its socket. Based on the unpacked message, UL_INFO struct on the VNF side gets populated. The NR_UL_indication() function is then called at the VNF, which uses UL_INFO.
Through the infinite loop [while(vnf_p7->terminate == 0)] running in nfapi_nr_vnf_p7_start(), the VNF receives and unpacks the uplink indication message received on its socket. Based on the unpacked messages, UL_INFO struct on the VNF side gets populated.
```
......@@ -68,17 +68,6 @@ Through the infinite loop [while(vnf_p7->terminate == 0)] running in nfapi_nr_vn
if(FD_ISSET(vnf_p7->socket, &rfds))
{
vnf_nr_p7_read_dispatch_message(vnf_p7);
// Call the scheduler
struct PHY_VARS_gNB_s *gNB = RC.gNB[0];
pthread_mutex_lock(&gNB->UL_INFO_mutex);
gNB->UL_INFO.frame = vnf_frame;
gNB->UL_INFO.slot = vnf_slot;
gNB->UL_INFO.module_id = gNB->Mod_id;
gNB->UL_INFO.CC_id = gNB->CC_id;
gNB->if_inst->NR_UL_indication(&gNB->UL_INFO);
pthread_mutex_unlock(&gNB->UL_INFO_mutex);
}
```
......@@ -87,7 +76,7 @@ vnf_nr_dispatch_p7_message() is the function that contains the switch on various
## P7 DL Transmission by VNF
DL messages are scheduled at the VNF, through NR_UL_indication() after processing of UL_INFO. The function performing the scheduling of channels (and consequently p7 messages in nFAPI) is gNB_dlsch_ulsch_scheduler().
DL messages are scheduled at the VNF, through NR_UL_indication(). NR_UL_indication() is called when the SFN/slot in the UL_info structure changes (this acts as a trigger for next slot processing, instead of running a separate clock at the VNF). The SFN/slot at the VNF in UL_info is updated using the slot_indication uplink p7 message, which is sent at the beginning of every slot by the PNF. The slot_indication message contains SFN/slot of the TX_thread, so that the scheduler operates slot_ahead slots ahead of the RX thread. This ensures that UL_tti_req is received before RX slot processing at the PNF.
The function NR_schedule_response calls oai_nfapi_[DL P7 msg]_req(), which contains the routines for packing + transmission of scheduled messages through the socket to the PNF. For example, to send the 'TX_data_req' p7 message
......@@ -109,12 +98,15 @@ graph TD
slot_inc -- next slot --> pselect
```
Note that since DL P7 message reception and TX/RX processing are done on separate threads, there is the issue of the L1 processing threads trying to do their job before the required P7 message is received. In the case of RX processing, since the scheduler operates slot_ahead slots ahead of the RX thread, the required messages conveniently arrive earlier than they are required. However, in the case of TX processing, this cannot be ensured if the scheduler is exactly in sync with the TX thread.
Therefore, we operate the VNF vnf_slot_ahead (which is currently 2) slots ahead of the PNF. This is to ensure that the DL fapi structures for a particular TX slot are all received before TX processing for that slot.
## P7 DL Reception at PNF
Through the infinite loop [while(pnf_p7->terminate == 0)] running in pnf_nr_p7_message_pump(), the PNF receives and unpacks the downlink P7 message received on its socket. Based on the unpacked message, the appropriate message structures are filled in the PNF, and these are used further down the pipeline for processing.
While receiving the DL P7 message, we check whether the message was received within a timing window from which it was sent. The duration of the window can be set by the user (set as a parameter for xnf in the p5 messages). A point to note is that the DL information must be received by the PNF within a timing window at least lesser than the duration of slot_ahead variable (timing_window <= slot_ahead * slot_duration). Ideally, it should be received within a duration significantly lesser than slot_ahead so that there is adequate time for PHY processing.
While receiving the DL P7 message, we check whether the message was received within a timing window from which it was sent. The duration of the window can be set by the user (set as a parameter for xnf in the p5 messages). Note that the DL information must be received by the PNF within a timing window at least lesser than the duration of slot_ahead variable (timing_window <= slot_ahead * slot_duration).
```mermaid
......@@ -124,7 +116,8 @@ graph TB
slot_inc -- next slot --> pselect
```
Once the messages are received, they are filled into slot buffers, and are stored until the processing of the slot that they were meant for.
......
......@@ -133,7 +133,7 @@ void tx_func(void *param) {
}
void rx_func(void *param) {
processingData_L1_t *info = (processingData_L1_t *) param;
PHY_VARS_gNB *gNB = info->gNB;
int frame_rx = info->frame_rx;
......@@ -142,7 +142,7 @@ void rx_func(void *param) {
int slot_tx = info->slot_tx;
sl_ahead = sf_ahead*gNB->frame_parms.slots_per_subframe;
nfapi_nr_config_request_scf_t *cfg = &gNB->gNB_config;
LOG_D(PHY,"slot rx %d \n", slot_rx);
start_meas(&softmodem_stats_rxtx_sf);
// *******************************************************************
......@@ -214,7 +214,6 @@ void rx_func(void *param) {
gNB->pucch[j]->pucch_pdu.rnti = 0;
pucch_removed++;
}
#if 0
for (j = 0; j < NUMBER_OF_NR_PDCCH_MAX; j++)
gNB->pdcch_pdu[j].frame = -1;
......@@ -253,7 +252,7 @@ void rx_func(void *param) {
stop_meas( &softmodem_stats_rxtx_sf );
LOG_D(PHY,"%s() Exit proc[rx:%d%d tx:%d%d]\n", __FUNCTION__, frame_rx, slot_rx, frame_tx, slot_tx);
// Call the scheduler
start_meas(&gNB->ul_indication_stats);
......@@ -265,7 +264,7 @@ void rx_func(void *param) {
gNB->if_inst->NR_UL_indication(&gNB->UL_INFO);
pthread_mutex_unlock(&gNB->UL_INFO_mutex);
stop_meas(&gNB->ul_indication_stats);
notifiedFIFO_elt_t *res;
if (tx_slot_type == NR_DOWNLINK_SLOT || tx_slot_type == NR_MIXED_SLOT) {
......
......@@ -2,7 +2,7 @@
## Conributed by 5G Testbed IISc
### Developers: Mahesh K,Gokul S,Aniq U R, Sai Shruthi N, Sudhakar B
### Developers: Gokul S, Mahesh A, Aniq U R
## Procedure to Build gNB and UE
......@@ -34,7 +34,21 @@ sudo ./nr-softmodem -O ../../../targets/PROJECTS/GENERIC-LTE-EPC/CONF/oaiL1.nfap
sudo RFSIMULATOR=127.0.0.1 ./nr-uesoftmodem --rfsim --phy-test --rrc_config_path . -d
```
## Procedure to run NR nFAPI using Hardware
## Procedure to run NR nFAPI using Hardware (tested with USRP x310)
To be updated.
### VNF command
```
sudo ./nr-softmodem -O ../../../targets/PROJECTS/GENERIC-LTE-EPC/CONF/rcc.band78.tm1.106PRB.nfapi.conf --nfapi 2 --noS1 --phy-test
```
### PNF command
```
sudo ./nr-softmodem -O ../../../targets/PROJECTS/GENERIC-LTE-EPC/CONF/oaiL1.nfapi.usrpx300.conf --nfapi 1 --phy-test
```
### UE command
```
sudo ./nr-uesoftmodem --usrp-args "addr=*USRP_ADDRESS*,clock_source=external,time_source=external" --phy-test --rrc_config_path ../../../ci-scripts/rrc-files
```
......@@ -37,8 +37,6 @@
#include "nfapi_pnf.h"
#include "common/ran_context.h"
#include "openair2/PHY_INTERFACE/phy_stub_UE.h"
//#include "openair1/PHY/vars.h"
extern RAN_CONTEXT_t RC;
#include <sys/socket.h>
#include <sys/time.h>
......@@ -67,6 +65,7 @@ extern RAN_CONTEXT_t RC;
extern void phy_init_RU(RU_t *);
extern int config_sync_var;
extern RAN_CONTEXT_t RC;
extern pthread_cond_t nfapi_sync_cond;
extern pthread_mutex_t nfapi_sync_mutex;
......@@ -110,8 +109,12 @@ extern void nr_fill_prach_ru(RU_t *ru,
int Slot,
nfapi_nr_prach_pdu_t *prach_pdu);
int nfapi_pnf_p7_nr_slot_ind(nfapi_pnf_p7_config_t* config, nfapi_nr_slot_indication_scf_t* ind);
int nfapi_pnf_p7_nr_rx_data_ind(nfapi_pnf_p7_config_t* config, nfapi_nr_rx_data_indication_t* ind);
int nfapi_pnf_p7_nr_crc_ind(nfapi_pnf_p7_config_t* config, nfapi_nr_crc_indication_t* ind);
int nfapi_pnf_p7_nr_srs_ind(nfapi_pnf_p7_config_t* config, nfapi_nr_srs_indication_t* ind);
int nfapi_pnf_p7_nr_uci_ind(nfapi_pnf_p7_config_t* config, nfapi_nr_uci_indication_t* ind);
int nfapi_pnf_p7_nr_rach_ind(nfapi_pnf_p7_config_t* config, nfapi_nr_rach_indication_t* ind);
nfapi_tx_request_pdu_t *tx_request_pdu[1023][10][10]; // [frame][subframe][max_num_pdus]
uint8_t nr_tx_pdus[32][16][4096];
......@@ -1146,16 +1149,13 @@ int pnf_phy_ul_dci_req(gNB_L1_rxtx_proc_t *proc, nfapi_pnf_p7_config_t *pnf_p7,
// LOG_D(PHY,"[PNF] HI_DCI0_REQUEST SFN/SF:%05d dci:%d hi:%d\n", NFAPI_SFNSF2DEC(req->sfn_sf), req->hi_dci0_request_body.number_of_dci, req->hi_dci0_request_body.number_of_hi);
//phy_info* phy = (phy_info*)(pnf_p7->user_data);
struct PHY_VARS_gNB_s *gNB = RC.gNB[0];
if (proc ==NULL)
proc = &gNB->proc.L1_proc;
//int slot_tx = req->Slot + 6; //Ref for PNF is slot_rx, so align slot number with tx thread before passing UL_DCI info
for (int i=0; i<req->numPdus; i++) {
//LOG_D(PHY,"[PNF] HI_DCI0_REQ sfn_sf:%d PDU[%d]\n", NFAPI_SFNSF2DEC(req->sfn_sf), i);
if (req->ul_dci_pdu_list[i].PDUType == 0) {
//LOG_D(PHY,"[PNF] HI_DCI0_REQ sfn_sf:%d PDU[%d] - NFAPI_HI_DCI0_DCI_PDU_TYPE\n", NFAPI_SFNSF2DEC(req->sfn_sf), i);
nfapi_nr_ul_dci_request_pdus_t *ul_dci_req_pdu = &req->ul_dci_pdu_list[i];
nfapi_nr_ul_dci_request_pdus_t *ul_dci_req_pdu = &req->ul_dci_pdu_list[i];
handle_nfapi_nr_ul_dci_pdu(gNB, req->SFN, req->Slot, ul_dci_req_pdu);
}
else {
......@@ -2318,7 +2318,7 @@ void handle_nr_slot_ind(uint16_t sfn, uint16_t slot) {
oai_nfapi_nr_slot_indication(ind);
//copy data from appropriate p7 slot buffers into channel structures for PHY processing
int slot_ret = nfapi_pnf_p7_slot_ind(p7_config_g, p7_config_g->phy_id, sfn, slot);
nfapi_pnf_p7_slot_ind(p7_config_g, p7_config_g->phy_id, sfn, slot);
return;
}
......
......@@ -19,8 +19,7 @@
* contact@openairinterface.org
*/
#if !defined(NFAPI_PNF_H__)
#define NFAPI_PNF_H__
extern nfapi_ue_release_request_body_t release_rntis;
int oai_nfapi_rach_ind(nfapi_rach_indication_t *rach_ind);
void configure_nfapi_pnf(char *vnf_ip_addr, int vnf_p5_port, char *pnf_ip_addr, int pnf_p7_port, int vnf_p7_port);
......@@ -28,4 +27,12 @@ void configure_nr_nfapi_pnf(char *vnf_ip_addr, int vnf_p5_port, char *pnf_ip_add
void oai_subframe_ind(uint16_t sfn, uint16_t sf);
void handle_nr_slot_ind(uint16_t sfn, uint16_t slot);
#endif
uint32_t sfnslot_add_slot(uint16_t sfn, uint16_t slot, int offset);
int oai_nfapi_nr_slot_indication(nfapi_nr_slot_indication_scf_t *ind);
int oai_nfapi_nr_rx_data_indication(nfapi_nr_rx_data_indication_t *ind);
int oai_nfapi_nr_crc_indication(nfapi_nr_crc_indication_t *ind);
int oai_nfapi_nr_srs_indication(nfapi_nr_srs_indication_t *ind);
int oai_nfapi_nr_uci_indication(nfapi_nr_uci_indication_t *ind);
int oai_nfapi_nr_rach_indication(nfapi_nr_rach_indication_t *ind);
......@@ -1014,31 +1014,6 @@ int phy_cqi_indication(struct nfapi_vnf_p7_config *config, nfapi_cqi_indication_
return 1;
}
// void add_slot(uint16_t *frameP, uint16_t *slotP, int offset)
// {
// uint16_t num_slots = 20; // set based on numerlogy (fixing for 1)
// *frameP = (*frameP + ((*slotP + offset) / num_slots))%1024;
// *slotP = ((*slotP + offset) % num_slots);
// }
// uint32_t sfnslot_add_slot(uint16_t sfn, uint16_t slot, int offset)
// {
// uint32_t new_sfnslot;
// // uint16_t sfn = NFAPI_SFNSLOT2SFN(sfnslot);
// // uint16_t slot = NFAPI_SFNSLOT2SLOT(sfnslot);
// //printf("%s() sfn:%u sf:%u\n", __FUNCTION__, sfn, sf);
// add_slot(&sfn, &slot, offset);
// new_sfnslot = sfn<<6|slot;
// //printf("%s() sfn:%u sf:%u offset:%d sfnsf:%d(DEC:%d) new:%d(DEC:%d)\n", __FUNCTION__, sfn, sf, offset, sfnsf, NFAPI_SFNSF2DEC(sfnsf), new_sfnsf, NFAPI_SFNSF2DEC(new_sfnsf));
// return new_sfnslot;
// }
//NR phy indication
int phy_nr_slot_indication(nfapi_nr_slot_indication_scf_t *ind) {
......@@ -1053,6 +1028,8 @@ int phy_nr_slot_indication(nfapi_nr_slot_indication_scf_t *ind) {
gNB->UL_INFO.slot = vnf_slot;
pthread_mutex_unlock(&gNB->UL_INFO_mutex);
LOG_D(MAC, "VNF SFN/Slot %d.%d \n", gNB->UL_INFO.frame, gNB->UL_INFO.slot);
return 1;
}
int phy_nr_crc_indication(nfapi_nr_crc_indication_t *ind) {
......
......@@ -19,9 +19,9 @@
* contact@openairinterface.org
*/
#if !defined(NFAPI_VNF_H__)
#define NFAPI_VNF_H__
void configure_nfapi_vnf(char *vnf_addr, int vnf_p5_port);
void configure_nr_nfapi_vnf(char *vnf_addr, int vnf_p5_port);
#endif
uint32_t sfnslot_add_slot(uint16_t sfn, uint16_t slot, int offset);
......@@ -1475,7 +1475,7 @@ typedef enum {
//section 3.4.7 rx_data_indication
///table 3-61
//table 3-61
typedef struct
{
......
......@@ -3636,16 +3636,13 @@ static uint8_t pack_nr_uci_indication_body(void* tlv, uint8_t **ppWritePackedMsg
printf("Unhandled NFAPI_NR_UCI_PUSCH_PDU_TYPE \n");
break;
case NFAPI_NR_UCI_FORMAT_0_1_PDU_TYPE: {
nfapi_nr_uci_pucch_pdu_format_0_1_t *uci_pdu = &value->pucch_pdu_format_0_1;
case NFAPI_NR_UCI_FORMAT_0_1_PDU_TYPE:
pack_nr_uci_pucch_0_1(&value->pucch_pdu_format_0_1, ppWritePackedMsg, end);
break;
}
case NFAPI_NR_UCI_FORMAT_2_3_4_PDU_TYPE: {
nfapi_nr_uci_pucch_pdu_format_2_3_4_t *uci_pdu = &value->pucch_pdu_format_2_3_4;
pack_nr_uci_pucch_2_3_4(uci_pdu, ppWritePackedMsg, end);
case NFAPI_NR_UCI_FORMAT_2_3_4_PDU_TYPE:
pack_nr_uci_pucch_2_3_4(&value->pucch_pdu_format_2_3_4, ppWritePackedMsg, end);
break;
}
}
return 1;
......@@ -6556,7 +6553,7 @@ return 1;
static uint8_t unpack_nr_uci_pucch_0_1(nfapi_nr_uci_pucch_pdu_format_0_1_t* tlv, uint8_t **ppReadPackedMsg, uint8_t *end) {
nfapi_nr_uci_pucch_pdu_format_0_1_t* value = (nfapi_nr_uci_pucch_pdu_format_0_1_t*)tlv;
uint8_t *ptr = *ppReadPackedMsg;
// uint8_t *ptr = *ppReadPackedMsg;
// printf("\n Read P7 message uci_0_1 indication unpack: ");
// while(ptr < end){
// printf(" %d ", *ptr);
......
......@@ -158,5 +158,7 @@ pnf_p7_rx_message_t* pnf_p7_rx_reassembly_queue_add_segment(pnf_p7_t* pnf_p7, pn
void pnf_p7_rx_reassembly_queue_remove_msg(pnf_p7_t* pnf_p7, pnf_p7_rx_reassembly_queue_t* queue, pnf_p7_rx_message_t* msg);
void pnf_p7_rx_reassembly_queue_remove_old_msgs(pnf_p7_t* pnf_p7, pnf_p7_rx_reassembly_queue_t* queue, uint32_t rx_hr_time, uint32_t delta);
int pnf_nr_p7_pack_and_send_p7_message(pnf_p7_t* pnf_p7, nfapi_p7_message_header_t* header, uint32_t msg_len);
#endif /* _PNF_P7_H_ */
......@@ -940,7 +940,7 @@ int pnf_p7_slot_ind(pnf_p7_t* pnf_p7, uint16_t phy_id, uint16_t sfn, uint16_t sl
//We align the pnf_p7 sfn/slot with tx sfn/slot, and vnf is synced with pnf_p7 sfn/slot. This is so that the scheduler runs slot_ahead from rx thread.
pnf_p7->sfn = sfn_tx; //gokul
pnf_p7->sfn = sfn_tx;
pnf_p7->slot = slot_tx;
uint32_t rx_slot_dec = NFAPI_SFNSLOT2DEC(sfn, slot);
......@@ -1142,7 +1142,7 @@ int pnf_p7_slot_ind(pnf_p7_t* pnf_p7, uint16_t phy_id, uint16_t sfn, uint16_t sl
else
{
pnf_p7->timing_info_ms_counter++;
} //gokul
}
}
......@@ -1772,7 +1772,7 @@ void pnf_handle_ul_tti_request(void* pRecvMsg, int recvMsgLen, pnf_p7_t* pnf_p7)
struct timespec t;
clock_gettime(CLOCK_MONOTONIC, &t);
//NFAPI_TRACE(NFAPI_TRACE_INFO,"%s() %ld.%09ld POPULATE UL_TTI_REQ current tx sfn/slot:%d.%d p7 msg sfn/slot: %d.%d buffer_index:%d\n", __FUNCTION__, t.tv_sec, t.tv_nsec, pnf_p7->sfn,pnf_p7->slot, req->SFN, req->Slot, buffer_index);
NFAPI_TRACE(NFAPI_TRACE_INFO,"%s() %ld.%09ld POPULATE UL_TTI_REQ current tx sfn/slot:%d.%d p7 msg sfn/slot: %d.%d buffer_index:%d\n", __FUNCTION__, t.tv_sec, t.tv_nsec, pnf_p7->sfn,pnf_p7->slot, req->SFN, req->Slot, buffer_index);
if(pnf_p7->slot_buffer[buffer_index].ul_tti_req != 0)
{
......
......@@ -175,7 +175,17 @@ int nfapi_pnf_p7_rach_ind(nfapi_pnf_p7_config_t* config, nfapi_rach_indication_t
pnf_p7_t* _this = (pnf_p7_t*)(config);
return pnf_p7_pack_and_send_p7_message(_this, (nfapi_p7_message_header_t*)ind, sizeof(nfapi_rach_indication_t));
}
int nfapi_pnf_p7_srs_ind(nfapi_pnf_p7_config_t* config, nfapi_srs_indication_t* ind)
{
if(config == NULL || ind == NULL)
{
NFAPI_TRACE(NFAPI_TRACE_ERROR, "%s: invalid input params\n", __FUNCTION__);
return -1;
}
pnf_p7_t* _this = (pnf_p7_t*)(config);
return pnf_p7_pack_and_send_p7_message(_this, (nfapi_p7_message_header_t*)ind, sizeof(nfapi_srs_indication_t));
}
int nfapi_pnf_p7_sr_ind(nfapi_pnf_p7_config_t* config, nfapi_sr_indication_t* ind)
{
if(config == NULL || ind == NULL)
......
......@@ -1473,7 +1473,7 @@ void vnf_handle_nr_slot_indication(void *pRecvMsg, int recvMsgLen, vnf_p7_t* vnf
{
if(vnf_p7->_public.nr_slot_indication)
{
(vnf_p7->_public.nr_slot_indication)(&ind); //gokul
(vnf_p7->_public.nr_slot_indication)(&ind);
}
}
......@@ -1982,7 +1982,7 @@ void vnf_nr_handle_ul_node_sync(void *pRecvMsg, int recvMsgLen, vnf_p7_t* vnf_p7
phy->zero_count,
phy->in_sync ? "IN_SYNC" : "OUT_OF_SYNC");*/
phy->sfn = new_sfn;
phy->sfn = new_sfn;
phy->slot = new_slot;
}
}
......@@ -1998,7 +1998,7 @@ void vnf_nr_handle_ul_node_sync(void *pRecvMsg, int recvMsgLen, vnf_p7_t* vnf_p7
phy->previous_t1 = ind.t1;
phy->previous_t2 = ind.t2;
}
}
}
void vnf_handle_timing_info(void *pRecvMsg, int recvMsgLen, vnf_p7_t* vnf_p7)
{
......@@ -2034,9 +2034,7 @@ void vnf_handle_timing_info(void *pRecvMsg, int recvMsgLen, vnf_p7_t* vnf_p7)
void vnf_nr_handle_timing_info(void *pRecvMsg, int recvMsgLen, vnf_p7_t* vnf_p7)
{
uint8_t uplink_slots[6] = {7,8,9,17,18,19};
uint8_t is_uplink_slot = 0;
{
if (pRecvMsg == NULL || vnf_p7 == NULL)
{
NFAPI_TRACE(NFAPI_TRACE_ERROR, "vnf_handle_timing_info: NULL parameters\n");
......@@ -2055,20 +2053,16 @@ void vnf_nr_handle_timing_info(void *pRecvMsg, int recvMsgLen, vnf_p7_t* vnf_p7)
//int16_t vnf_pnf_sfnsf_delta = NFAPI_SFNSF2DEC(vnf_p7->p7_connections[0].sfn_sf) - NFAPI_SFNSF2DEC(ind.last_sfn_sf);
int16_t vnf_pnf_sfnslot_delta = NFAPI_SFNSLOT2DEC(vnf_p7->p7_connections[0].sfn,vnf_p7->p7_connections[0].slot) - NFAPI_SFNSLOT2DEC(ind.last_sfn,ind.last_slot);
//NFAPI_TRACE(NFAPI_TRACE_INFO, "%s() PNF:SFN/SF:%d VNF:SFN/SF:%d deltaSFNSF:%d\n", __FUNCTION__, NFAPI_SFNSF2DEC(ind.last_sfn_sf), NFAPI_SFNSF2DEC(vnf_p7->p7_connections[0].sfn_sf), vnf_pnf_sfnsf_delta);
// Panos: Careful here!!! Modification of the original nfapi-code
//if (vnf_pnf_sfnsf_delta>1 || vnf_pnf_sfnsf_delta < -1)
//printf("VNF-PNF delta - %d", vnf_pnf_sfnslot_delta);
for (int i = 0; i < 6; i++){
if (vnf_p7->p7_connections[0].slot == uplink_slots[i])
is_uplink_slot = 1;
}
if ((vnf_pnf_sfnslot_delta>0 || vnf_pnf_sfnslot_delta < 0) && is_uplink_slot == 0)
if (vnf_pnf_sfnslot_delta>0 || vnf_pnf_sfnslot_delta < 0)
{
NFAPI_TRACE(NFAPI_TRACE_INFO, "%s() LARGE SFN/SF DELTA between PNF and VNF delta:%d VNF:%d PNF:%d\n\n\n\n\n\n\n\n\n", __FUNCTION__, vnf_pnf_sfnslot_delta,NFAPI_SFNSLOT2DEC(vnf_p7->p7_connections[0].sfn,vnf_p7->p7_connections[0].slot),NFAPI_SFNSLOT2DEC(ind.last_sfn,ind.last_slot)) ;
// Panos: Careful here!!! Modification of the original nfapi-code
vnf_p7->p7_connections[0].sfn = ind.last_sfn;
vnf_p7->p7_connections[0].slot = ind.last_slot;
//printf("VNF SFN/slot modified");
// vnf_p7->p7_connections[0].sfn = ind.last_sfn;
// vnf_p7->p7_connections[0].slot = ind.last_slot;
}
}
}
......
......@@ -99,8 +99,7 @@ struct timespec timespec_sub(struct timespec lhs, struct timespec rhs)
int nfapi_nr_vnf_p7_start(nfapi_vnf_p7_config_t* config)
{
struct PHY_VARS_gNB_s *gNB = RC.gNB[0];
uint8_t temp_slot;
uint16_t vnf_frame; uint8_t vnf_slot;
uint8_t prev_slot = 0;
if(config == 0)
return -1;
......@@ -174,7 +173,6 @@ int nfapi_nr_vnf_p7_start(nfapi_vnf_p7_config_t* config)
struct timespec slot_start;
// clock_gettime(CLOCK_MONOTONIC, &sf_start);
clock_gettime(CLOCK_MONOTONIC, &slot_start);
long millisecond = slot_start.tv_nsec / 1e6; //Check if we have to change
//long millisecond = slot_start.tv_nsec / 0.5e6;
// sf_start = timespec_add(sf_start, sf_duration);
slot_start = timespec_add(slot_start, slot_duration);
......@@ -216,7 +214,7 @@ int nfapi_nr_vnf_p7_start(nfapi_vnf_p7_config_t* config)
pselect_timeout = timespec_sub(slot_start, pselect_start);
}
if(setup_time > 10 && temp_slot != gNB->UL_INFO.slot){
if(setup_time > 10 && prev_slot != gNB->UL_INFO.slot){
//Call the scheduler
pthread_mutex_lock(&gNB->UL_INFO_mutex);
......@@ -224,7 +222,7 @@ int nfapi_nr_vnf_p7_start(nfapi_vnf_p7_config_t* config)
gNB->UL_INFO.CC_id = gNB->CC_id;
gNB->if_inst->NR_UL_indication(&gNB->UL_INFO);
pthread_mutex_unlock(&gNB->UL_INFO_mutex);
temp_slot = gNB->UL_INFO.slot;
prev_slot = gNB->UL_INFO.slot;
}
selectRetval = pselect(maxSock+1, &rfds, NULL, NULL, &pselect_timeout, NULL);
......@@ -266,8 +264,6 @@ int nfapi_nr_vnf_p7_start(nfapi_vnf_p7_config_t* config)
curr->slot++;
}
//printf("Frame = %d, slot = %d in VNF main loop. \n",curr->sfn,curr->slot);
vnf_frame = curr->sfn;
vnf_slot = curr->slot;
//vnf_nr_sync(vnf_p7, curr);
......
......@@ -780,11 +780,11 @@ int dlsch_encoding_SIC(PHY_VARS_UE *ue,
for (r=0; r<dlsch->harq_processes[harq_pid]->C; r++) {
#ifdef DEBUG_DLSCH_CODING
// printf("Rate Matching, Code segment %u (coded bits (G) %u,unpunctured/repeated bits per code segment %u,mod_order %d, nb_rb %d)...\n",
// r,
// G,
// Kr*3,
// mod_order,nb_rb);
printf("Rate Matching, Code segment %u (coded bits (G) %u,unpunctured/repeated bits per code segment %u,mod_order %d, nb_rb %d)...\n",
r,
G,
Kr*3,
mod_order,nb_rb);
#endif
start_meas(rm_stats);
#ifdef DEBUG_DLSCH_CODING
......
......@@ -211,8 +211,8 @@ int16_t find_nr_pdcch(int frame,int slot, PHY_VARS_gNB *gNB,find_type_t type) {
for (i=0; i<NUMBER_OF_NR_PDCCH_MAX; i++) {
LOG_D(PHY,"searching for frame.slot %d.%d : pdcch_index %d frame.slot %d.%d, first_free_index %d\n", frame,slot,i,gNB->pdcch_pdu[i].frame,gNB->pdcch_pdu[i].slot,first_free_index);
if ((gNB->pdcch_pdu[i].frame == frame) &&
(gNB->pdcch_pdu[i].slot==slot)) {return i;}
else if ( gNB->pdcch_pdu[i].frame==-1 && first_free_index==-1) {first_free_index=i;}
(gNB->pdcch_pdu[i].slot==slot)) return i;
else if ( gNB->pdcch_pdu[i].frame==-1 && first_free_index==-1) first_free_index=i;
}
if (type == SEARCH_EXIST) return -1;
......@@ -227,7 +227,7 @@ void nr_fill_dci(PHY_VARS_gNB *gNB,
nfapi_nr_dl_tti_pdcch_pdu_rel15_t *pdcch_pdu_rel15 = &pdcch_pdu->pdcch_pdu_rel15;
NR_gNB_DLSCH_t *dlsch;
//printf("find pdcch from nr_fill_dci \n");
int pdcch_id = find_nr_pdcch(frame,slot,gNB,SEARCH_EXIST_OR_FREE);
AssertFatal(pdcch_id>=0 && pdcch_id<NUMBER_OF_NR_PDCCH_MAX,"Cannot find space for PDCCH, exiting\n");
memcpy((void*)&gNB->pdcch_pdu[pdcch_id].pdcch_pdu,(void*)pdcch_pdu,sizeof(*pdcch_pdu));
......@@ -285,7 +285,7 @@ void nr_fill_ul_dci(PHY_VARS_gNB *gNB,
nfapi_nr_ul_dci_request_pdus_t *pdcch_pdu) {
nfapi_nr_dl_tti_pdcch_pdu_rel15_t *pdcch_pdu_rel15 = &pdcch_pdu->pdcch_pdu.pdcch_pdu_rel15;
//printf("find ul dci from fill_ul_dci \n");
int pdcch_id = find_nr_ul_dci(frame,slot,gNB,SEARCH_EXIST_OR_FREE);
AssertFatal(pdcch_id>=0 && pdcch_id<NUMBER_OF_NR_PDCCH_MAX,"Cannot find space for UL PDCCH, exiting\n");
memcpy((void*)&gNB->ul_pdcch_pdu[pdcch_id].pdcch_pdu,(void*)pdcch_pdu,sizeof(*pdcch_pdu));
......
......@@ -292,7 +292,7 @@ int nr_dlsch_encoding(PHY_VARS_gNB *gNB,
}
G = nr_get_G(nb_rb, nb_symb_sch, nb_re_dmrs, length_dmrs,mod_order,rel15->nrOfLayers);
LOG_I(NR_PHY,"dlsch coding A %d G %d (nb_rb %d, nb_symb_sch %d, nb_re_dmrs %d, length_dmrs %d, mod_order %d)\n", A,G, nb_rb,nb_symb_sch,nb_re_dmrs,length_dmrs,mod_order);
LOG_D(NR_PHY,"dlsch coding A %d G %d (nb_rb %d, nb_symb_sch %d, nb_re_dmrs %d, length_dmrs %d, mod_order %d)\n", A,G, nb_rb,nb_symb_sch,nb_re_dmrs,length_dmrs,mod_order);
if (A > 3824) {
// Add 24-bit crc (polynomial A) to payload
......
......@@ -431,7 +431,7 @@ void nr_processULSegment(void* arg) {
if (check_crc((uint8_t*)llrProcBuf,length_dec,ulsch_harq->F,crc_type)) {
#ifdef PRINT_CRC_CHECK
LOG_D(PHY, "Segment %d CRC OK\n",r);
LOG_I(PHY, "Segment %d CRC OK\n",r);
#endif
rdata->decodeIterations = no_iteration_ldpc;
} else {
......
......@@ -88,8 +88,6 @@ void nr_fill_pucch(PHY_VARS_gNB *gNB,
int frame,
int slot,
nfapi_nr_pucch_pdu_t *pucch_pdu) {
// if (NFAPI_MODE == NFAPI_MODE_PNF)
// gNB->pucch[0]->active = 0;
int id = nr_find_pucch(pucch_pdu->rnti,frame,slot,gNB);
AssertFatal( (id>=0) && (id<NUMBER_OF_NR_PUCCH_MAX),
......
......@@ -104,7 +104,7 @@ void handle_nfapi_nr_pdcch_pdu(PHY_VARS_gNB *gNB,
int frame, int slot,
nfapi_nr_dl_tti_pdcch_pdu *pdcch_pdu) {
LOG_D(PHY,"Frame %d, Slot %d: DCI processing - pdcch_pdu_rel15->numDlDci:%d\n",frame,slot, pdcch_pdu->pdcch_pdu_rel15.numDlDci);
LOG_D(PHY,"Frame %d, Slot %d: DCI processing - proc:slot_tx:%d pdcch_pdu_rel15->numDlDci:%d\n",frame,slot, slot, pdcch_pdu->pdcch_pdu_rel15.numDlDci);
// copy dci configuration into gNB structure
// gNB->pdcch_pdu = pdcch_pdu;
......@@ -118,7 +118,7 @@ void handle_nfapi_nr_ul_dci_pdu(PHY_VARS_gNB *gNB,
int frame, int slot,
nfapi_nr_ul_dci_request_pdus_t *ul_dci_request_pdu) {
LOG_D(PHY,"Frame %d, Slot %d: UL DCI processing - pdcch_pdu_rel15->numDlDci:%d\n",frame,slot, ul_dci_request_pdu->pdcch_pdu.pdcch_pdu_rel15.numDlDci);
LOG_D(PHY,"Frame %d, Slot %d: UL DCI processing - proc:slot_tx:%d pdcch_pdu_rel15->numDlDci:%d\n",frame,slot, slot, ul_dci_request_pdu->pdcch_pdu.pdcch_pdu_rel15.numDlDci);
// copy dci configuration into gNB structure
// gNB->ul_dci_pdu = ul_dci_request_pdu;
......@@ -203,39 +203,35 @@ void nr_schedule_response(NR_Sched_Rsp_t *Sched_INFO){
LOG_D(PHY,"NFAPI: dl_pdu %d : type %d\n",i,dl_tti_pdu->PDUType);
switch (dl_tti_pdu->PDUType) {
case NFAPI_NR_DL_TTI_SSB_PDU_TYPE:
handle_nr_nfapi_ssb_pdu(gNB,frame,slot,
handle_nr_nfapi_ssb_pdu(gNB,frame,slot,
dl_tti_pdu);
break;
break;
case NFAPI_NR_DL_TTI_PDCCH_PDU_TYPE:
AssertFatal(pdcch_received == 0, "pdcch_received is not 0, we can only handle one PDCCH PDU per slot\n");
handle_nfapi_nr_pdcch_pdu(gNB,
frame, slot,
&dl_tti_pdu->pdcch_pdu);
pdcch_received = 1;
break;
break;
case NFAPI_NR_DL_TTI_CSI_RS_PDU_TYPE:
LOG_D(PHY,"frame %d, slot %d, Got NFAPI_NR_DL_TTI_CSI_RS_PDU_TYPE for %d.%d\n",frame,slot,DL_req->SFN,DL_req->Slot);
handle_nfapi_nr_csirs_pdu(gNB,
frame, slot,
&dl_tti_pdu->csi_rs_pdu);
break;
break;
case NFAPI_NR_DL_TTI_PDSCH_PDU_TYPE:
LOG_D(PHY,"frame %d, slot %d, Got NFAPI_NR_DL_TTI_PDSCH_PDU_TYPE for %d.%d\n",frame,slot,DL_req->SFN,DL_req->Slot);
nfapi_nr_dl_tti_pdsch_pdu_rel15_t *pdsch_pdu_rel15 = &dl_tti_pdu->pdsch_pdu.pdsch_pdu_rel15;
uint16_t pduIndex = pdsch_pdu_rel15->pduIndex;
AssertFatal(TX_req->pdu_list[pduIndex].num_TLV == 1, "TX_req->pdu_list[%d].num_TLV %d != 1\n",
pduIndex,TX_req->pdu_list[pduIndex].num_TLV);
uint8_t *sdu = (uint8_t *)TX_req->pdu_list[pduIndex].TLVs[0].value.direct;
if(NFAPI_MODE != NFAPI_MODE_VNF)
handle_nr_nfapi_pdsch_pdu(gNB,frame,slot,&dl_tti_pdu->pdsch_pdu, sdu);
LOG_D(PHY,"frame %d, slot %d, Got NFAPI_NR_DL_TTI_PDSCH_PDU_TYPE for %d.%d\n",frame,slot,DL_req->SFN,DL_req->Slot);
nfapi_nr_dl_tti_pdsch_pdu_rel15_t *pdsch_pdu_rel15 = &dl_tti_pdu->pdsch_pdu.pdsch_pdu_rel15;
uint16_t pduIndex = pdsch_pdu_rel15->pduIndex;
AssertFatal(TX_req->pdu_list[pduIndex].num_TLV == 1, "TX_req->pdu_list[%d].num_TLV %d != 1\n",
pduIndex,TX_req->pdu_list[pduIndex].num_TLV);
uint8_t *sdu = (uint8_t *)TX_req->pdu_list[pduIndex].TLVs[0].value.direct;
if(NFAPI_MODE != NFAPI_MODE_VNF)
handle_nr_nfapi_pdsch_pdu(gNB,frame,slot,&dl_tti_pdu->pdsch_pdu, sdu);
break;
}
}
......
......@@ -148,15 +148,15 @@ void phy_procedures_gNB_TX(PHY_VARS_gNB *gNB,
VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_PHY_PROCEDURES_gNB_COMMON_TX,1);
for (int i=0; i<fp->Lmax; i++) {
if (gNB->ssb[i].active) {
nr_common_signal_procedures(gNB,frame,slot,gNB->ssb[i].ssb_pdu);
gNB->ssb[i].active = false;
}
for (int i=0; i<fp->Lmax; i++) {
if (gNB->ssb[i].active) {
nr_common_signal_procedures(gNB,frame,slot,gNB->ssb[i].ssb_pdu);
gNB->ssb[i].active = false;
}
}
VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_PHY_PROCEDURES_gNB_COMMON_TX,0);
//printf("find pdcch from TX func \n");
int pdcch_pdu_id=find_nr_pdcch(frame,slot,gNB,SEARCH_EXIST);
int ul_pdcch_pdu_id=find_nr_ul_dci(frame,slot,gNB,SEARCH_EXIST);
......@@ -568,7 +568,6 @@ int phy_procedures_gNB_uespec_RX(PHY_VARS_gNB *gNB, int frame_rx, int slot_rx) {
for (int i=0;i<NUMBER_OF_NR_PUCCH_MAX;i++){
NR_gNB_PUCCH_t *pucch = gNB->pucch[i];
if (pucch) {
LOG_D(PHY, "SFN/slot = %d.%d, pucch[%d]->active = %d \n", frame_rx, slot_rx, i,pucch->active);
if ((pucch->active == 1) &&
(pucch->frame == frame_rx) &&
(pucch->slot == slot_rx) ) {
......@@ -598,9 +597,7 @@ int phy_procedures_gNB_uespec_RX(PHY_VARS_gNB *gNB, int frame_rx, int slot_rx) {
pucch_pdu);
gNB->UL_INFO.uci_ind.num_ucis += 1;
//printf("SFN/slot %d.%d pucch active of index %d set to 0. \n", frame_rx, slot_rx, i);
pucch->active = 0;
//printf("Decode pucch num ucis +=1 in %d.%d \n",frame_rx,slot_rx);
break;
case 2:
num_ucis = gNB->UL_INFO.uci_ind.num_ucis;
......@@ -617,7 +614,6 @@ int phy_procedures_gNB_uespec_RX(PHY_VARS_gNB *gNB, int frame_rx, int slot_rx) {
pucch_pdu);
gNB->UL_INFO.uci_ind.num_ucis += 1;
//printf("SFN/slot %d.%d pucch active of index %d set to 0. \n", frame_rx, slot_rx, i);
pucch->active = 0;
break;
default:
......
......@@ -60,7 +60,7 @@
uint16_t nr_pdcch_order_table[6] = { 31, 31, 511, 2047, 2047, 8191 };
uint8_t first_sched_entry = 1;
uint8_t vnf_first_sched_entry = 1;
void clear_mac_stats(gNB_MAC_INST *gNB) {
memset((void*)gNB->UE_info.mac_stats,0,MAX_MOBILES_PER_GNB*sizeof(NR_mac_stats_t));
......@@ -400,18 +400,19 @@ void gNB_dlsch_ulsch_scheduler(module_id_t module_idP,
clear_nr_nfapi_information(RC.nrmac[module_idP], CC_id, frame, slot);
//first entry
/*VNF first entry into scheduler. Since frame numbers for future_ul_tti_req of some future slots
will not be set before we encounter them, set them here */
if (NFAPI_MODE == NFAPI_MODE_VNF){
if(first_sched_entry == 1)
if(vnf_first_sched_entry == 1)
{
printf("First sched entry at slot %d \n", slot);
for (int i = 0; i<num_slots; i++){
if(i < slot)
gNB->UL_tti_req_ahead[CC_id][i].SFN = (frame + 1) % 1024;
else
gNB->UL_tti_req_ahead[CC_id][i].SFN = frame;
}
first_sched_entry = 0;
vnf_first_sched_entry = 0;
}
}
}
......@@ -442,7 +443,7 @@ void gNB_dlsch_ulsch_scheduler(module_id_t module_idP,
schedule_nr_prach(module_idP, f, s);
}
// This schedule SR
// This schedule SR
nr_sr_reporting(module_idP, frame, slot);
// Schedule CSI-RS transmission
......@@ -459,7 +460,6 @@ void gNB_dlsch_ulsch_scheduler(module_id_t module_idP,
}
// This schedules the DCI for Uplink and subsequently PUSCH
nr_schedule_ulsch(module_idP, frame, slot);
// This schedules the DCI for Downlink and PDSCH
......
......@@ -1036,7 +1036,6 @@ static NR_UE_harq_t *find_harq(module_id_t mod_id, frame_t frame, sub_frame_t sl
harq->feedback_slot,
frame,
slot);
//printf("handling harq \n");
remove_front_nr_list(&sched_ctrl->feedback_dl_harq);
handle_dl_harq(mod_id, UE_id, pid, 0);
pid = sched_ctrl->feedback_dl_harq.head;
......@@ -1076,14 +1075,12 @@ void handle_nr_uci_pucch_0_1(module_id_t mod_id,
for (int harq_bit = 0; harq_bit < uci_01->harq->num_harq; harq_bit++) {
const uint8_t harq_value = uci_01->harq->harq_list[harq_bit].harq_value;
const uint8_t harq_confidence = uci_01->harq->harq_confidence_level;
//printf("handle uci 0_1 find harq SFN/slot %d.%d \n",frame,slot);
NR_UE_harq_t *harq = find_harq(mod_id, frame, slot, UE_id);
if (!harq)
break;
DevAssert(harq->is_waiting);
const int8_t pid = sched_ctrl->feedback_dl_harq.head;
remove_front_nr_list(&sched_ctrl->feedback_dl_harq);
//printf("handling harq \n");
handle_dl_harq(mod_id, UE_id, pid, harq_value == 1 && harq_confidence == 0);
}
}
......@@ -1152,6 +1149,7 @@ void handle_nr_uci_pucch_2_3_4(module_id_t mod_id,
}
}
// function to update pucch scheduling parameters in UE list when a USS DL is scheduled
// this function returns an index to NR_sched_pucch structure
// currently this structure contains PUCCH0 at index 0 and PUCCH2 at index 1
......@@ -1193,7 +1191,6 @@ int nr_acknack_scheduling(int mod_id,
__func__,
pucch->csi_bits);
/* if the currently allocated PUCCH of this UE is full, allocate it */
if (pucch->dai_c == 2) {
/* advance the UL slot information in PUCCH by one so we won't schedule in
* the same slot again */
......@@ -1223,7 +1220,6 @@ int nr_acknack_scheduling(int mod_id,
/* if the UE's next PUCCH occasion is after the possible UL slots (within the
* same frame) or wrapped around to the next frame, then we assume there is
* no possible PUCCH allocation anymore */
if ((pucch->frame == frame
&& (pucch->ul_slot >= first_ul_slot_tdd + nr_ulmix_slots))
|| (pucch->frame == frame + 1))
......@@ -1235,10 +1231,8 @@ int nr_acknack_scheduling(int mod_id,
get_pdsch_to_harq_feedback(mod_id, UE_id, ss_type, pdsch_to_harq_feedback);
/* there is a HARQ. Check whether we can use it for this ACKNACK */
if (pucch->dai_c > 0) {
/* this UE already has a PUCCH occasion */
LOG_I(MAC,"pucch->frame = %d,frame=%d \n",pucch->frame,frame);
DevAssert(pucch->frame == frame);
// Find the right timing_indicator value.
......
......@@ -32,7 +32,6 @@
#include "LAYER2/NR_MAC_gNB/mac_proto.h"
#include "executables/softmodem-common.h"
#include "common/utils/nr/nr_common.h"
#include "nfapi/oai_integration/vendor_ext.h"
#include <openair2/UTIL/OPT/opt.h>
......@@ -1346,11 +1345,9 @@ void nr_schedule_ulsch(module_id_t module_id, frame_t frame, sub_frame_t slot)
sched_ctrl->SR = false;
int8_t harq_id = sched_pusch->ul_harq_pid;
if (harq_id < 0) {
/* PP has not selected a specific HARQ Process, get a new one */
harq_id = sched_ctrl->available_ul_harq.head;
//printf("SFN/slot %d.%d harq id = %d \n", frame, slot, harq_id);
AssertFatal(harq_id >= 0,
"no free HARQ process available for UE %d\n",
UE_id);
......@@ -1358,8 +1355,8 @@ void nr_schedule_ulsch(module_id_t module_id, frame_t frame, sub_frame_t slot)
sched_pusch->ul_harq_pid = harq_id;
} else {
/* PP selected a specific HARQ process. Check whether it will be a new
* transmission or a retransmission, and remove from the corresponding
* list */
* transmission or a retransmission, and remove from the corresponding
* list */
if (sched_ctrl->ul_harq_processes[harq_id].round == 0)
remove_nr_list(&sched_ctrl->available_ul_harq, harq_id);
else
......@@ -1370,7 +1367,6 @@ void nr_schedule_ulsch(module_id_t module_id, frame_t frame, sub_frame_t slot)
add_tail_nr_list(&sched_ctrl->feedback_ul_harq, harq_id);
cur_harq->feedback_slot = sched_pusch->slot;
cur_harq->is_waiting = true;
int rnti_types[2] = { NR_RNTI_C, 0 };
......
......@@ -48,6 +48,12 @@ extern int oai_nfapi_crc_indication(nfapi_crc_indication_t *crc_ind);
extern int oai_nfapi_cqi_indication(nfapi_cqi_indication_t *cqi_ind);
extern int oai_nfapi_sr_indication(nfapi_sr_indication_t *ind);
extern int oai_nfapi_rx_ind(nfapi_rx_indication_t *ind);
int oai_nfapi_nr_slot_indication(nfapi_nr_slot_indication_scf_t *ind);
int oai_nfapi_nr_rx_data_indication(nfapi_nr_rx_data_indication_t *ind);
int oai_nfapi_nr_crc_indication(nfapi_nr_crc_indication_t *ind);
int oai_nfapi_nr_srs_indication(nfapi_nr_srs_indication_t *ind);
int oai_nfapi_nr_uci_indication(nfapi_nr_uci_indication_t *ind);
int oai_nfapi_nr_rach_indication(nfapi_nr_rach_indication_t *ind);
extern uint8_t nfapi_mode;
extern uint16_t sf_ahead;
extern uint16_t sl_ahead;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment