Commit 6102fa4b authored by Cedric Roux's avatar Cedric Roux

- Fix some performances issues for softmodem

- Remove eventfd if task has only polling mode
- Created dummy thread to convert eventfd from RT thread to non RT threads
- Added eNB app to softmodem
- Set poll mode for itti dumper thread in softmodem
- Disable timers for softmodem

git-svn-id: http://svn.eurecom.fr/openair4G/trunk@4545 818b1a75-f10b-46b9-bf7c-635c3b92a50f
parent 18f113c4
This diff is collapsed.
...@@ -38,6 +38,10 @@ ...@@ -38,6 +38,10 @@
# include <sys/epoll.h> # include <sys/epoll.h>
#endif #endif
#ifdef RTAI
# include <rtai_sem.h>
#endif
#ifndef INTERTASK_INTERFACE_H_ #ifndef INTERTASK_INTERFACE_H_
#define INTERTASK_INTERFACE_H_ #define INTERTASK_INTERFACE_H_
...@@ -91,6 +95,7 @@ typedef enum task_priorities_e { ...@@ -91,6 +95,7 @@ typedef enum task_priorities_e {
typedef struct task_info_s { typedef struct task_info_s {
thread_id_t thread; thread_id_t thread;
task_id_t parent_task;
task_priorities_t priority; task_priorities_t priority;
unsigned int queue_size; unsigned int queue_size;
/* Printable name */ /* Printable name */
......
...@@ -56,9 +56,9 @@ const char * const messages_definition_xml = { ...@@ -56,9 +56,9 @@ const char * const messages_definition_xml = {
/* Map task id to printable name. */ /* Map task id to printable name. */
const task_info_t tasks_info[] = { const task_info_t tasks_info[] = {
{0, 0, 0, "TASK_UNKNOWN"}, {0, TASK_UNKNOWN, 0, 0, "TASK_UNKNOWN"},
#define TASK_DEF(tHREADiD, pRIO, qUEUEsIZE) { tHREADiD##_THREAD, pRIO, qUEUEsIZE, #tHREADiD }, #define TASK_DEF(tHREADiD, pRIO, qUEUEsIZE) { tHREADiD##_THREAD, TASK_UNKNOWN, pRIO, qUEUEsIZE, #tHREADiD },
#define SUB_TASK_DEF(tHREADiD, sUBtASKiD, qUEUEsIZE) { sUBtASKiD##_THREAD, 0, qUEUEsIZE, #sUBtASKiD }, #define SUB_TASK_DEF(tHREADiD, sUBtASKiD, qUEUEsIZE) { sUBtASKiD##_THREAD, tHREADiD##_THREAD, 0, qUEUEsIZE, #sUBtASKiD },
#include <tasks_def.h> #include <tasks_def.h>
#undef SUB_TASK_DEF #undef SUB_TASK_DEF
#undef TASK_DEF #undef TASK_DEF
......
...@@ -57,6 +57,7 @@ ...@@ -57,6 +57,7 @@
/* Defines to extract task ID fields */ /* Defines to extract task ID fields */
#define TASK_GET_THREAD_ID(tASKiD) (itti_desc.tasks_info[tASKiD].thread) #define TASK_GET_THREAD_ID(tASKiD) (itti_desc.tasks_info[tASKiD].thread)
#define TASK_GET_PARENT_TASK_ID(tASKiD) (itti_desc.tasks_info[tASKiD].parent_task)
/* Extract the instance from a message */ /* Extract the instance from a message */
#define ITTI_MESSAGE_GET_INSTANCE(mESSAGE) ((mESSAGE)->ittiMsgHeader.instance) #define ITTI_MESSAGE_GET_INSTANCE(mESSAGE) ((mESSAGE)->ittiMsgHeader.instance)
......
...@@ -80,6 +80,7 @@ do { \ ...@@ -80,6 +80,7 @@ do { \
int timer_handle_signal(siginfo_t *info) int timer_handle_signal(siginfo_t *info)
{ {
#if !defined(RTAI)
struct timer_elm_s *timer_p; struct timer_elm_s *timer_p;
MessageDef *message_p; MessageDef *message_p;
timer_has_expired_t *timer_expired_p; timer_has_expired_t *timer_expired_p;
...@@ -121,6 +122,7 @@ int timer_handle_signal(siginfo_t *info) ...@@ -121,6 +122,7 @@ int timer_handle_signal(siginfo_t *info)
free(message_p); free(message_p);
return -1; return -1;
} }
#endif
return 0; return 0;
} }
...@@ -134,6 +136,7 @@ int timer_setup( ...@@ -134,6 +136,7 @@ int timer_setup(
void *timer_arg, void *timer_arg,
long *timer_id) long *timer_id)
{ {
#if !defined(RTAI)
struct sigevent se; struct sigevent se;
struct itimerspec its; struct itimerspec its;
struct timer_elm_s *timer_p; struct timer_elm_s *timer_p;
...@@ -201,14 +204,16 @@ int timer_setup( ...@@ -201,14 +204,16 @@ int timer_setup(
pthread_mutex_lock(&timer_desc.timer_list_mutex); pthread_mutex_lock(&timer_desc.timer_list_mutex);
STAILQ_INSERT_TAIL(&timer_desc.timer_queue, timer_p, entries); STAILQ_INSERT_TAIL(&timer_desc.timer_queue, timer_p, entries);
pthread_mutex_unlock(&timer_desc.timer_list_mutex); pthread_mutex_unlock(&timer_desc.timer_list_mutex);
#endif
return 0; return 0;
} }
int timer_remove(long timer_id) int timer_remove(long timer_id)
{ {
struct timer_elm_s *timer_p;
int rc = 0; int rc = 0;
#if !defined(RTAI)
struct timer_elm_s *timer_p;
TMR_DEBUG("Removing timer 0x%lx\n", timer_id); TMR_DEBUG("Removing timer 0x%lx\n", timer_id);
...@@ -231,6 +236,7 @@ int timer_remove(long timer_id) ...@@ -231,6 +236,7 @@ int timer_remove(long timer_id)
} }
free(timer_p); free(timer_p);
timer_p = NULL; timer_p = NULL;
#endif
return rc; return rc;
} }
......
...@@ -4,7 +4,7 @@ TASK_DEF(TASK_TIMER, TASK_PRIORITY_MAX, 10) ...@@ -4,7 +4,7 @@ TASK_DEF(TASK_TIMER, TASK_PRIORITY_MAX, 10)
// Other possible tasks in the process // Other possible tasks in the process
/// Layer 2 and Layer 1 task supporting all the synchronous processing /// Layer 2 and Layer 1 task supporting all the synchronous processing
TASK_DEF(TASK_L2L1, TASK_PRIORITY_MAX_LEAST, 10) TASK_DEF(TASK_L2L1, TASK_PRIORITY_MAX_LEAST, 1000)
//// Layer 2 and Layer 1 sub-tasks //// Layer 2 and Layer 1 sub-tasks
SUB_TASK_DEF(TASK_L2L1, TASK_PDCP_UE, 200) SUB_TASK_DEF(TASK_L2L1, TASK_PDCP_UE, 200)
SUB_TASK_DEF(TASK_L2L1, TASK_PDCP_ENB, 200) SUB_TASK_DEF(TASK_L2L1, TASK_PDCP_ENB, 200)
......
...@@ -79,7 +79,13 @@ const char* eurecomVariablesNames[] = { ...@@ -79,7 +79,13 @@ const char* eurecomVariablesNames[] = {
"frame_number", "frame_number",
"slot_number", "slot_number",
"daq_mbox", "daq_mbox",
"diff2" "diff2",
"itti_send_msg",
"itti_send_msg_end",
"itti_poll_msg",
"itti_poll_msg_end",
"itti_recv_msg",
"itti_recv_msg_end"
}; };
const char* eurecomFunctionsNames[] = { const char* eurecomFunctionsNames[] = {
...@@ -154,6 +160,7 @@ const char* eurecomFunctionsNames[] = { ...@@ -154,6 +160,7 @@ const char* eurecomFunctionsNames[] = {
"phy_eNB_dlsch_scramblig", "phy_eNB_dlsch_scramblig",
"pdcp_apply_security", "pdcp_apply_security",
"pdcp_validate_security", "pdcp_validate_security",
"itti_dump_enqueue_message",
"test" "test"
}; };
......
...@@ -49,6 +49,12 @@ typedef enum ...@@ -49,6 +49,12 @@ typedef enum
VCD_SIGNAL_DUMPER_VARIABLES_SLOT_NUMBER, VCD_SIGNAL_DUMPER_VARIABLES_SLOT_NUMBER,
VCD_SIGNAL_DUMPER_VARIABLES_DAQ_MBOX, VCD_SIGNAL_DUMPER_VARIABLES_DAQ_MBOX,
VCD_SIGNAL_DUMPER_VARIABLES_DIFF, VCD_SIGNAL_DUMPER_VARIABLES_DIFF,
VCD_SIGNAL_DUMPER_VARIABLE_ITTI_SEND_MSG,
VCD_SIGNAL_DUMPER_VARIABLE_ITTI_SEND_MSG_END,
VCD_SIGNAL_DUMPER_VARIABLE_ITTI_POLL_MSG,
VCD_SIGNAL_DUMPER_VARIABLE_ITTI_POLL_MSG_END,
VCD_SIGNAL_DUMPER_VARIABLE_ITTI_RECV_MSG,
VCD_SIGNAL_DUMPER_VARIABLE_ITTI_RECV_MSG_END,
VCD_SIGNAL_DUMPER_VARIABLES_LAST, VCD_SIGNAL_DUMPER_VARIABLES_LAST,
VCD_SIGNAL_DUMPER_VARIABLES_END = VCD_SIGNAL_DUMPER_VARIABLES_LAST, VCD_SIGNAL_DUMPER_VARIABLES_END = VCD_SIGNAL_DUMPER_VARIABLES_LAST,
} vcd_signal_dump_variables; } vcd_signal_dump_variables;
...@@ -126,6 +132,7 @@ typedef enum ...@@ -126,6 +132,7 @@ typedef enum
VCD_SIGNAL_DUMPER_FUNCTIONS_ENB_DLSCH_SCRAMBLING, VCD_SIGNAL_DUMPER_FUNCTIONS_ENB_DLSCH_SCRAMBLING,
VCD_SIGNAL_DUMPER_FUNCTIONS_PDCP_APPLY_SECURITY, VCD_SIGNAL_DUMPER_FUNCTIONS_PDCP_APPLY_SECURITY,
VCD_SIGNAL_DUMPER_FUNCTIONS_PDCP_VALIDATE_SECURITY, VCD_SIGNAL_DUMPER_FUNCTIONS_PDCP_VALIDATE_SECURITY,
VCD_SIGNAL_DUMPER_FUNCTIONS_ITTI_DUMP_ENQUEUE_MESSAGE,
VCD_SIGNAL_DUMPER_FUNCTIONS_TEST, VCD_SIGNAL_DUMPER_FUNCTIONS_TEST,
VCD_SIGNAL_DUMPER_FUNCTIONS_LAST, VCD_SIGNAL_DUMPER_FUNCTIONS_LAST,
VCD_SIGNAL_DUMPER_FUNCTIONS_END = VCD_SIGNAL_DUMPER_FUNCTIONS_LAST, VCD_SIGNAL_DUMPER_FUNCTIONS_END = VCD_SIGNAL_DUMPER_FUNCTIONS_LAST,
......
...@@ -98,7 +98,7 @@ CFLAGS += -DOPENAIR2 -DNO_RRM -DPUCCH -DMAC_CONTEXT=1 ...@@ -98,7 +98,7 @@ CFLAGS += -DOPENAIR2 -DNO_RRM -DPUCCH -DMAC_CONTEXT=1
endif endif
ifdef ENABLE_ITTI ifdef ENABLE_ITTI
OBJ += $(UTILS_OBJS) RTAI_OBJ += $(UTILS_OBJS)
endif endif
CFLAGS += $(L2_incl) $(UTIL_incl) $(UTILS_incl) CFLAGS += $(L2_incl) $(UTIL_incl) $(UTILS_incl)
...@@ -174,6 +174,12 @@ $(LFDS_LIB): ...@@ -174,6 +174,12 @@ $(LFDS_LIB):
$(RTAI_OBJ): %.o : %.c $(RTAI_OBJ): %.o : %.c
@echo Compiling $< ... @echo Compiling $< ...
@$(CC) -c $(CFLAGS) $(EXTRA_CFLAGS) $(RTAI_CFLAGS) -o $@ $< @$(CC) -c $(CFLAGS) $(EXTRA_CFLAGS) $(RTAI_CFLAGS) -o $@ $<
@$(CC) -MM $(CFLAGS) $(EXTRA_CFLAGS) $(RTAI_CFLAGS) $< > $*.d
@mv -f $*.d $*.d.tmp
@sed -e 's|.*:|$*.o:|' < $*.d.tmp > $*.d
@sed -e 's/.*://' -e 's/\\$$//' < $*.d.tmp | fmt -1 | \
sed -e 's/^ *//' -e 's/$$/:/' >> $*.d
@rm -f $*.d.tmp
ifdef ENABLE_ITTI ifdef ENABLE_ITTI
$(OBJ) $(RTAI_OBJ): $(ITTI_MESSAGES_H) $(OBJ) $(RTAI_OBJ): $(ITTI_MESSAGES_H)
...@@ -201,7 +207,7 @@ synctest: $(OBJ) $(ASN1_MSG_OBJS1) $(RTAI_OBJ) ...@@ -201,7 +207,7 @@ synctest: $(OBJ) $(ASN1_MSG_OBJS1) $(RTAI_OBJ)
lte-softmodem: $(OBJ) $(ASN1_MSG_OBJS1) $(RTAI_OBJ) $(SHARED_DEPENDENCIES) lte-softmodem: $(OBJ) $(ASN1_MSG_OBJS1) $(RTAI_OBJ) $(SHARED_DEPENDENCIES)
@echo Linking $@ @echo Linking $@
$(CC) $(CFLAGS) $(EXTRA_CFLAGS) $(OBJ) $(RTAI_OBJ) $(ASN1_MSG_OBJS1) -o lte-softmodem $(LDFLAGS) $(LIBS) @$(CC) $(CFLAGS) $(EXTRA_CFLAGS) $(OBJ) $(RTAI_OBJ) $(ASN1_MSG_OBJS1) -o lte-softmodem $(LDFLAGS) $(LIBS)
lte-softmodem-usrp: $(OBJ) $(ASN1_MSG_OBJS1) $(RTAI_OBJ) $(USRP_OBJ) $(SHARED_DEPENDENCIES) lte-softmodem-usrp: $(OBJ) $(ASN1_MSG_OBJS1) $(RTAI_OBJ) $(USRP_OBJ) $(SHARED_DEPENDENCIES)
@echo Linking $@ @echo Linking $@
......
...@@ -52,6 +52,7 @@ ...@@ -52,6 +52,7 @@
#include <getopt.h> #include <getopt.h>
#include "rt_wrapper.h" #include "rt_wrapper.h"
#include "assertions.h"
#ifdef EMOS #ifdef EMOS
#include <gps.h> #include <gps.h>
...@@ -148,7 +149,7 @@ exmimo_config_t *p_exmimo_config; ...@@ -148,7 +149,7 @@ exmimo_config_t *p_exmimo_config;
exmimo_id_t *p_exmimo_id; exmimo_id_t *p_exmimo_id;
volatile unsigned int *DAQ_MBOX; volatile unsigned int *DAQ_MBOX;
int oai_exit = 0; volatile int oai_exit = 0;
//int time_offset[4] = {-138,-138,-138,-138}; //int time_offset[4] = {-138,-138,-138,-138};
//int time_offset[4] = {-145,-145,-145,-145}; //int time_offset[4] = {-145,-145,-145,-145};
...@@ -456,6 +457,32 @@ void *emos_thread (void *arg) ...@@ -456,6 +457,32 @@ void *emos_thread (void *arg)
} }
#endif #endif
#if defined(ENABLE_ITTI)
void *dummy_l2l1_task(void *arg)
{
ssize_t ret = 0;
MessageDef *received_msg;
itti_mark_task_ready(TASK_L2L1);
while (!oai_exit)
{
usleep(100);
do {
itti_poll_msg(TASK_L2L1, &received_msg);
if (received_msg != NULL) {
itti_send_msg_to_task(ITTI_MSG_DESTINATION_ID(received_msg),
ITTI_MSG_INSTANCE(received_msg),
received_msg);
}
} while(received_msg != NULL);
}
return NULL;
}
#endif
/* This is the main eNB thread. It gets woken up by the kernel driver using the RTAI message mechanism (rt_send and rt_receive). */ /* This is the main eNB thread. It gets woken up by the kernel driver using the RTAI message mechanism (rt_send and rt_receive). */
static void *eNB_thread(void *arg) static void *eNB_thread(void *arg)
{ {
...@@ -510,8 +537,12 @@ static void *eNB_thread(void *arg) ...@@ -510,8 +537,12 @@ static void *eNB_thread(void *arg)
// at the eNB, even slots have double as much time since most of the processing is done here and almost nothing in odd slots // at the eNB, even slots have double as much time since most of the processing is done here and almost nothing in odd slots
LOG_D(HW,"eNB Frame %d, time %llu: missed slot, proceeding with next one (slot %d, hw_slot %d, diff %d)\n",frame, rt_get_time_ns(), slot, hw_slot, diff); LOG_D(HW,"eNB Frame %d, time %llu: missed slot, proceeding with next one (slot %d, hw_slot %d, diff %d)\n",frame, rt_get_time_ns(), slot, hw_slot, diff);
slot++; slot++;
if (frame>0) if (frame>0) {
oai_exit=1; oai_exit=1;
#if defined(ENABLE_ITTI)
itti_send_terminate_message(TASK_L2L1);
#endif
}
if (slot==20){ if (slot==20){
slot=0; slot=0;
frame++; frame++;
...@@ -915,17 +946,10 @@ static void *UE_thread(void *arg) ...@@ -915,17 +946,10 @@ static void *UE_thread(void *arg)
return 0; return 0;
} }
/* DUmmy l2l1 task */ void *eNB_app_task(void *args_p)
void *l2l1_task(void *args_p) { {
#if defined(ENABLE_ITTI) && defined(ENABLE_USE_MME)
#if defined(ENABLE_ITTI)
MessageDef *message_p; MessageDef *message_p;
itti_mark_task_ready (TASK_L2L1);
# if defined(ENABLE_USE_MME)
/* Trying to register each eNB */
{
char *mme_address_v4; char *mme_address_v4;
char *mme_address_v6 = "2001:660:5502:12:30da:829a:2343:b6cf"; char *mme_address_v6 = "2001:660:5502:12:30da:829a:2343:b6cf";
s1ap_register_eNB_t *s1ap_register_eNB; s1ap_register_eNB_t *s1ap_register_eNB;
...@@ -941,7 +965,7 @@ void *l2l1_task(void *args_p) { ...@@ -941,7 +965,7 @@ void *l2l1_task(void *args_p) {
} }
/* FIXME: following parameters should be setup by eNB applicative layer ? */ /* FIXME: following parameters should be setup by eNB applicative layer ? */
message_p = itti_alloc_new_message(TASK_L2L1, S1AP_REGISTER_ENB); message_p = itti_alloc_new_message(TASK_ENB_APP, S1AP_REGISTER_ENB);
s1ap_register_eNB = &message_p->ittiMsg.s1ap_register_eNB; s1ap_register_eNB = &message_p->ittiMsg.s1ap_register_eNB;
...@@ -963,14 +987,12 @@ void *l2l1_task(void *args_p) { ...@@ -963,14 +987,12 @@ void *l2l1_task(void *args_p) {
strlen(mme_address_v6)); strlen(mme_address_v6));
itti_send_msg_to_task(TASK_S1AP, eNB_id, message_p); itti_send_msg_to_task(TASK_S1AP, eNB_id, message_p);
}
# endif
#endif
#if defined(ENABLE_ITTI) itti_mark_task_ready (TASK_ENB_APP); // at the end of init for the current task
while (1) {
do {
// Checks if a message has been sent to L2L1 task // Checks if a message has been sent to L2L1 task
itti_receive_msg (TASK_L2L1, &message_p); itti_receive_msg (TASK_ENB_APP, &message_p);
if (message_p != NULL) { if (message_p != NULL) {
switch (ITTI_MSG_ID(message_p)) { switch (ITTI_MSG_ID(message_p)) {
...@@ -989,7 +1011,7 @@ void *l2l1_task(void *args_p) { ...@@ -989,7 +1011,7 @@ void *l2l1_task(void *args_p) {
free (message_p); free (message_p);
} }
} } while(1);
#endif #endif
return NULL; return NULL;
...@@ -1205,6 +1227,13 @@ int main(int argc, char **argv) { ...@@ -1205,6 +1227,13 @@ int main(int argc, char **argv) {
// initialize the log (see log.h for details) // initialize the log (see log.h for details)
logInit(); logInit();
if (ouput_vcd) {
if (UE_flag==1)
vcd_signal_dumper_init("/tmp/openair_dump_UE.vcd");
else
vcd_signal_dumper_init("/tmp/openair_dump_eNB.vcd");
}
#if defined(ENABLE_ITTI) #if defined(ENABLE_ITTI)
itti_init(TASK_MAX, THREAD_MAX, MESSAGES_ID_MAX, tasks_info, messages_info, messages_definition_xml, itti_dump_file); itti_init(TASK_MAX, THREAD_MAX, MESSAGES_ID_MAX, tasks_info, messages_info, messages_definition_xml, itti_dump_file);
...@@ -1219,9 +1248,14 @@ int main(int argc, char **argv) { ...@@ -1219,9 +1248,14 @@ int main(int argc, char **argv) {
LOG_D(EMU, "Initializing S1AP task interface: FAILED\n"); LOG_D(EMU, "Initializing S1AP task interface: FAILED\n");
return -1; return -1;
} }
if (itti_create_task(TASK_ENB_APP, eNB_app_task, NULL) < 0) {
LOG_E(EMU, "Create task failed");
LOG_D(EMU, "Initializing eNB APP task interface: FAILED\n");
return -1;
}
# endif # endif
if (itti_create_task(TASK_L2L1, l2l1_task, NULL) < 0) { if (itti_create_task(TASK_L2L1, dummy_l2l1_task, NULL) < 0) {
LOG_E(EMU, "Create task failed"); LOG_E(EMU, "Create task failed");
LOG_D(EMU, "Initializing L2L1 task interface: FAILED\n"); LOG_D(EMU, "Initializing L2L1 task interface: FAILED\n");
return -1; return -1;
...@@ -1231,13 +1265,6 @@ int main(int argc, char **argv) { ...@@ -1231,13 +1265,6 @@ int main(int argc, char **argv) {
// itti_wait_tasks_end(); // itti_wait_tasks_end();
#endif #endif
if (ouput_vcd) {
if (UE_flag==1)
vcd_signal_dumper_init("/tmp/openair_dump_UE.vcd");
else
vcd_signal_dumper_init("/tmp/openair_dump_eNB.vcd");
}
#ifdef NAS_NETLINK #ifdef NAS_NETLINK
netlink_init(); netlink_init();
#endif #endif
...@@ -1419,7 +1446,12 @@ int main(int argc, char **argv) { ...@@ -1419,7 +1446,12 @@ int main(int argc, char **argv) {
g_log->log_component[OTG].flag = LOG_HIGH; g_log->log_component[OTG].flag = LOG_HIGH;
g_log->log_component[RRC].level = LOG_INFO; g_log->log_component[RRC].level = LOG_INFO;
g_log->log_component[RRC].flag = LOG_HIGH; g_log->log_component[RRC].flag = LOG_HIGH;
#if defined(ENABLE_ITTI) && defined(ENABLE_USE_MME)
g_log->log_component[S1AP].level = LOG_INFO;
g_log->log_component[S1AP].flag = LOG_HIGH;
g_log->log_component[SCTP].level = LOG_INFO;
g_log->log_component[SCTP].flag = LOG_HIGH;
#endif
PHY_vars_eNB_g = malloc(sizeof(PHY_VARS_eNB*)); PHY_vars_eNB_g = malloc(sizeof(PHY_VARS_eNB*));
PHY_vars_eNB_g[0] = init_lte_eNB(frame_parms,eNB_id,Nid_cell,cooperation_flag,transmission_mode,abstraction_flag); PHY_vars_eNB_g[0] = init_lte_eNB(frame_parms,eNB_id,Nid_cell,cooperation_flag,transmission_mode,abstraction_flag);
......
...@@ -485,10 +485,9 @@ void *eNB_app_task(void *args_p) { ...@@ -485,10 +485,9 @@ void *eNB_app_task(void *args_p) {
} while(1); } while(1);
itti_terminate_tasks(TASK_ENB_APP); itti_terminate_tasks(TASK_ENB_APP);
#endif
return NULL; return NULL;
#endif
return NULL;
} }
void *l2l1_task(void *args_p) { void *l2l1_task(void *args_p) {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment