lte-softmodem.c

/*******************************************************************************

  Eurecom OpenAirInterface
  Copyright(c) 1999 - 2011 Eurecom

  This program is free software; you can redistribute it and/or modify it
  under the terms and conditions of the GNU General Public License,
  version 2, as published by the Free Software Foundation.

  This program is distributed in the hope it will be useful, but WITHOUT
  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  more details.

  You should have received a copy of the GNU General Public License along with
  this program; if not, write to the Free Software Foundation, Inc.,
  51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.

  The full GNU General Public License is included in this distribution in
  the file called "COPYING".

  Contact Information
  Openair Admin: openair_admin@eurecom.fr
  Openair Tech : openair_tech@eurecom.fr
  Forums       : http://forums.eurecom.fsr/openairinterface
  Address      : Eurecom, 2229, route des crêtes, 06560 Valbonne Sophia Antipolis, France

*******************************************************************************/

/*! \file lte-softmodem.c
 * \brief main program to control HW and scheduling
 * \author R. Knopp, F. Kaltenberger
 * \date 2012
 * \version 0.1
 * \company Eurecom
 * \email: knopp@eurecom.fr,florian.kaltenberger@eurecom.fr
 * \note
 * \warning
 */
#define _GNU_SOURCE
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
#include <sys/ioctl.h>
#include <sys/types.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <sched.h>
#include <signal.h>
#include <execinfo.h>
#include <getopt.h>

#include "rt_wrapper.h"
#undef MALLOC //there are two conflicting definitions, so we better make sure we don't use it at all

#ifdef USRP
static int hw_subframe;
#endif

#include "assertions.h"

#ifdef EMOS
#include <gps.h>
#endif

#include "PHY/types.h"

#include "PHY/defs.h"
#undef MALLOC //there are two conflicting definitions, so we better make sure we don't use it at all
#undef FRAME_LENGTH_COMPLEX_SAMPLES //there are two conflicting definitions, so we better make sure we don't use it at all

#ifndef USRP
#include "openair0_lib.h"
#else
#include "../../ARCH/COMMON/common_lib.h"
#endif

#undef FRAME_LENGTH_COMPLEX_SAMPLES //there are two conflicting definitions, so we better make sure we don't use it at all

#include "PHY/vars.h"
#include "MAC_INTERFACE/vars.h"
//#include "SCHED/defs.h"
#include "SCHED/vars.h"
#include "LAYER2/MAC/vars.h"

#include "../../SIMU/USER/init_lte.h"

#ifdef EMOS
#include "SCHED/phy_procedures_emos.h"
#endif

#ifdef OPENAIR2
#include "otg_tx.h"
#include "LAYER2/MAC/defs.h"
#include "LAYER2/MAC/vars.h"
#include "LAYER2/MAC/proto.h"
#ifndef CELLULAR
#include "RRC/LITE/vars.h"
#endif
#include "PHY_INTERFACE/vars.h"
#endif

#ifdef SMBV
#include "PHY/TOOLS/smbv.h"
unsigned short config_frames[4] = {2,9,11,13};
#endif
#include "UTIL/LOG/log_extern.h"
#include "UTIL/OTG/otg.h"
#include "UTIL/OTG/otg_vars.h"
#include "UTIL/MATH/oml.h"
#include "UTIL/LOG/vcd_signal_dumper.h"
#include "enb_config.h"

#if defined(ENABLE_ITTI)
# include "intertask_interface_init.h"
# include "create_tasks.h"
# if defined(ENABLE_USE_MME)
#   include "s1ap_eNB.h"
# endif
#endif

#ifdef XFORMS
#include "PHY/TOOLS/lte_phy_scope.h"
#include "stats.h"
#endif

#define FRAME_PERIOD    100000000ULL
#define DAQ_PERIOD      66667ULL

#define DEBUG_THREADS 1

struct timing_info_t {
  //unsigned int frame, hw_slot, last_slot, next_slot;
  RTIME time_min, time_max, time_avg, time_last, time_now;
  //unsigned int mbox0, mbox1, mbox2, mbox_target;
  unsigned int n_samples;
} timing_info;

extern int16_t* sync_corr_ue0;
extern int16_t prach_ifft[4][1024*2];

int init_dlsch_threads(void);
void cleanup_dlsch_threads(void);
int32_t init_rx_pdsch_thread(void);
void cleanup_rx_pdsch_thread(void);


int32_t *rxdata;
int32_t *txdata;
void setup_ue_buffers(PHY_VARS_UE *phy_vars_ue, LTE_DL_FRAME_PARMS *frame_parms, int carrier);
void setup_eNB_buffers(PHY_VARS_eNB *phy_vars_eNB, LTE_DL_FRAME_PARMS *frame_parms, int carrier);

#ifdef XFORMS
// current status is that every UE has a DL scope for a SINGLE eNB (eNB_id=0)
// at eNB 0, an UL scope for every UE 
FD_lte_phy_scope_ue  *form_ue[NUMBER_OF_UE_MAX];
FD_lte_phy_scope_enb *form_enb[NUMBER_OF_UE_MAX];
FD_stats_form                  *form_stats=NULL;
char title[255];
unsigned char                   scope_enb_num_ue = 1;
#endif //XFORMS

#ifdef RTAI
static SEM                     *mutex;
//static CND *cond;

static int                      thread0;
static int                      thread1;
#ifdef USRP
static SEM *sync_sem; // to sync rx & tx streaming
#endif
//static int sync_thread;
#else
pthread_t                       thread0;
pthread_t                       thread1;
pthread_attr_t                  attr_dlsch_threads;
struct sched_param              sched_param_dlsch;
#ifdef USRP
pthread_cond_t sync_cond;
pthread_mutex_t sync_mutex;
#endif
#endif

pthread_attr_t                  attr_eNB_proc_tx[10];
pthread_attr_t                  attr_eNB_proc_rx[10];
struct sched_param              sched_param_eNB_proc_tx[10];
struct sched_param              sched_param_eNB_proc_rx[10];

#ifdef XFORMS
static pthread_t                thread2; //xforms
#endif
#ifdef EMOS
static pthread_t                thread3; //emos
#endif

openair0_device openair0;
openair0_timestamp timestamp;

/*
  static int instance_cnt=-1; //0 means worker is busy, -1 means its free
  int instance_cnt_ptr_kern,*instance_cnt_ptr_user;
  int pci_interface_ptr_kern;
*/
//extern unsigned int bigphys_top;
//extern unsigned int mem_base;

int                             card = 0;


#if defined(ENABLE_ITTI)
static volatile int             start_eNB = 0;
static volatile int             start_UE = 0;
#endif
volatile int                    oai_exit = 0;

//static int                      time_offset[4] = {-138,-138,-138,-138};
//static int                      time_offset[4] = {-145,-145,-145,-145};
static int                      time_offset[4] = {0,0,0,0};


static char                     UE_flag=0;
static uint8_t                       eNB_id=0,UE_id=0;

uint32_t                             carrier_freq[4] =           {1907600000,1907600000,1907600000,1907600000}; /* For UE! */
static uint32_t          downlink_frequency[4] =     {1907600000,1907600000,1907600000,1907600000};
static int32_t                      uplink_frequency_offset[4]= {-120000000,-120000000,-120000000,-120000000};
static char                    *conf_config_file_name = NULL;

#if defined(ENABLE_ITTI)
static char                    *itti_dump_file = NULL;
#endif

double tx_gain = 20;
double rx_gain = 20;

double sample_rate=30.72e6;
double bw = 14e6;

#ifndef USRP
static unsigned int             rxg_max[4] =    {133,133,133,133};
static unsigned int             rxg_med[4] =    {127,127,127,127};
static unsigned int             rxg_byp[4] =    {120,120,120,120};
static int                      tx_max_power =  0;


#else
static unsigned int             rxg_max[4] =    {133,133,133,133};
//static unsigned int            rxg_med[4] =    {127,127,127,127};
//static unsigned int            rxg_byp[4] =    {120,120,120,120};
static int                      tx_max_power =  0;
char ref[128] = "internal";
char channels[128] = "0";

int samples_per_frame = 307200;
int samples_per_packets = 2048; // samples got every recv or send
int tx_forward_nsamps;

int sf_bounds_5[10] = {8, 15, 23, 30, 38, 45, 53, 60, 68, 75};
int sf_bounds_10[10] = {8, 15, 23, 30, 38, 45, 53, 60, 68, 75};
int sf_bounds_20[10] = {15, 30, 45, 60, 75, 90, 105, 120, 135, 150};
int *sf_bounds;
int max_cnt;
int tx_delay;

#endif
/*
  uint32_t rf_mode_max[4]     = {55759,55759,55759,55759};
  uint32_t rf_mode_med[4]     = {39375,39375,39375,39375};
  uint32_t rf_mode_byp[4]     = {22991,22991,22991,22991};
*/
//static uint32_t                      rf_mode[4] =        {MY_RF_MODE,0,0,0};
//static uint32_t                      rf_local[4] =       {8255000,8255000,8255000,8255000}; // UE zepto
//{8254617, 8254617, 8254617, 8254617}; //eNB khalifa
//{8255067,8254810,8257340,8257340}; // eNB PETRONAS

//static uint32_t                      rf_vcocal[4] =      {910,910,910,910};
//static uint32_t                      rf_vcocal_850[4] =  {2015, 2015, 2015, 2015};
//static uint32_t                      rf_rxdc[4] =        {32896,32896,32896,32896};
//static uint32_t                      rxgain[4] =         {20,20,20,20};
//static uint32_t                      txgain[4] =         {20,20,20,20};

static runmode_t                mode;
static int                      rx_input_level_dBm;
static int                      online_log_messages=0;
#ifdef XFORMS
extern int                      otg_enabled;
static char                     do_forms=0;
#else
int                             otg_enabled;
#endif
//int                             number_of_cards =   1;
#ifndef USRP
static int                      mbox_bounds[20] =   {8,16,24,30,38,46,54,60,68,76,84,90,98,106,114,120,128,136,144, 0}; ///boundaries of slots in terms ob mbox counter rounded up to even numbers
//static int                      mbox_bounds[20] =   {6,14,22,28,36,44,52,58,66,74,82,88,96,104,112,118,126,134,142, 148}; ///boundaries of slots in terms ob mbox counter rounded up to even numbers
#endif

static LTE_DL_FRAME_PARMS      *frame_parms;

int multi_thread=1;
int N_RB_DL=25;

unsigned int build_rflocal(int txi, int txq, int rxi, int rxq)
{
  return (txi + (txq<<6) + (rxi<<12) + (rxq<<18));
}
unsigned int build_rfdc(int dcoff_i_rxfe, int dcoff_q_rxfe)
{
  return (dcoff_i_rxfe + (dcoff_q_rxfe<<8));
}

#if !defined(ENABLE_ITTI)
void signal_handler(int sig)
{
  void *array[10];
  size_t size;

  if (sig==SIGSEGV) {
    // get void*'s for all entries on the stack
    size = backtrace(array, 10);
    
    // print out all the frames to stderr
    fprintf(stderr, "Error: signal %d:\n", sig);
    backtrace_symbols_fd(array, size, 2);
    exit(-1);
  }
  else {
    oai_exit = 1;
  }
}
#endif

void exit_fun(const char* s)
{
  if (s != NULL) {
    printf("%s %s() Exiting: %s\n",__FILE__, __FUNCTION__, s);
  }

  oai_exit = 1;

#if defined(ENABLE_ITTI)
  itti_terminate_tasks (TASK_UNKNOWN);
#endif

  //rt_sleep_ns(FRAME_PERIOD);

  //exit (-1);
}

static int latency_target_fd = -1;
static int32_t latency_target_value = 0;
/* Latency trick - taken from cyclictest.c 
* if the file /dev/cpu_dma_latency exists,
* open it and write a zero into it. This will tell
* the power management system not to transition to
* a high cstate (in fact, the system acts like idle=poll)
* When the fd to /dev/cpu_dma_latency is closed, the behavior
* goes back to the system default.
*
* Documentation/power/pm_qos_interface.txt
*/
static void set_latency_target(void)
{
  struct stat s;
  int ret;
  if (stat("/dev/cpu_dma_latency", &s) == 0) {
    latency_target_fd = open("/dev/cpu_dma_latency", O_RDWR);
    if (latency_target_fd == -1)
      return;
    ret = write(latency_target_fd, &latency_target_value, 4);
    if (ret == 0) {
      printf("# error setting cpu_dma_latency to %d!: %s\n", latency_target_value, strerror(errno));
      close(latency_target_fd);
      return;
    }
    printf("# /dev/cpu_dma_latency set to %dus\n", latency_target_value);
  }
}

#ifdef XFORMS
static void *scope_thread(void *arg) {
  char stats_buffer[16384];
# ifdef ENABLE_XFORMS_WRITE_STATS
  FILE *UE_stats, *eNB_stats;
  int len = 0;
# endif
  struct sched_param sched_param;

  sched_param.sched_priority = sched_get_priority_min(SCHED_FIFO)+1; 
  sched_setscheduler(0, SCHED_FIFO,&sched_param);

  printf("Scope thread has priority %d\n",sched_param.sched_priority);
    
  /*
    if (UE_flag==1) 
    UE_stats  = fopen("UE_stats.txt", "w");
    else 
    eNB_stats = fopen("eNB_stats.txt", "w");
  */
    
  while (!oai_exit) {
    if (UE_flag==1) {
# ifdef ENABLE_XFORMS_WRITE_STATS
      len =
# endif
	dump_ue_stats (PHY_vars_UE_g[0], stats_buffer, 0, mode,rx_input_level_dBm);
      fl_set_object_label(form_stats->stats_text, stats_buffer);
# ifdef ENABLE_XFORMS_WRITE_STATS
      rewind (UE_stats);
      fwrite (stats_buffer, 1, len, UE_stats);
# endif
      phy_scope_UE(form_ue[UE_id], 
		   PHY_vars_UE_g[UE_id],
		   eNB_id,
		   UE_id,7);
            
    } else {
# ifdef ENABLE_XFORMS_WRITE_STATS
      len =
# endif
	dump_eNB_stats (PHY_vars_eNB_g[0], stats_buffer, 0);
      fl_set_object_label(form_stats->stats_text, stats_buffer);
# ifdef ENABLE_XFORMS_WRITE_STATS
      rewind (eNB_stats);
      fwrite (stats_buffer, 1, len, eNB_stats);
# endif
      for(UE_id=0;UE_id<scope_enb_num_ue;UE_id++) {
	phy_scope_eNB(form_enb[UE_id], 
		      PHY_vars_eNB_g[eNB_id],
		      UE_id);
      }
              
    }
    //printf("doing forms\n");
    usleep(100000); // 100 ms
  }
    
# ifdef ENABLE_XFORMS_WRITE_STATS
  fclose (UE_stats);
  fclose (eNB_stats);
# endif
    
  pthread_exit((void*)arg);
}
#endif

#ifdef EMOS
#define NO_ESTIMATES_DISK 100 //No. of estimates that are aquired before dumped to disk

void *emos_thread (void *arg)
{
  char c;
  char *fifo2file_buffer, *fifo2file_ptr;

  int fifo, counter=0, bytes;

  FILE  *dumpfile_id;
  char  dumpfile_name[1024];
  time_t starttime_tmp;
  struct tm starttime;
  
  int channel_buffer_size;
  
  time_t timer;
  struct tm *now;

  struct gps_data_t *gps_data = NULL;
  struct gps_fix_t dummy_gps_data;

  struct sched_param sched_param;
  
  sched_param.sched_priority = sched_get_priority_max(SCHED_FIFO)-1; 
  sched_setscheduler(0, SCHED_FIFO,&sched_param);
  
  printf("EMOS thread has priority %d\n",sched_param.sched_priority);
 
  timer = time(NULL);
  now = localtime(&timer);

  memset(&dummy_gps_data,1,sizeof(struct gps_fix_t));
  
  gps_data = gps_open("127.0.0.1","2947");
  if (gps_data == NULL) 
    {
      printf("[EMOS] Could not open GPS\n");
      //exit(-1);
    }
#if GPSD_API_MAJOR_VERSION>=4
  else if (gps_stream(gps_data, WATCH_ENABLE,NULL) != 0)
#else
  else if (gps_query(gps_data, "w+x") != 0)
#endif
    {
      //sprintf(tmptxt,"Error sending command to GPS, gps_data = %x", gps_data);
      printf("[EMOS] Error sending command to GPS\n");
      //exit(-1);
    }
  else 
    printf("[EMOS] Opened GPS, gps_data=%p\n");
  
  if (UE_flag==0)
    channel_buffer_size = sizeof(fifo_dump_emos_eNB);
  else
    channel_buffer_size = sizeof(fifo_dump_emos_UE);

  // allocate memory for NO_FRAMES_DISK channes estimations
  fifo2file_buffer = malloc(NO_ESTIMATES_DISK*channel_buffer_size);
  fifo2file_ptr = fifo2file_buffer;

  if (fifo2file_buffer == NULL)
    {
      printf("[EMOS] Cound not allocate memory for fifo2file_buffer\n");
      exit(EXIT_FAILURE);
    }

  if ((fifo = open(CHANSOUNDER_FIFO_DEV, O_RDONLY)) < 0)
    {
      fprintf(stderr, "[EMOS] Error opening the fifo\n");
      exit(EXIT_FAILURE);
    }


  time(&starttime_tmp);
  localtime_r(&starttime_tmp,&starttime);
  snprintf(dumpfile_name,1024,"/tmp/%s_data_%d%02d%02d_%02d%02d%02d.EMOS",
	   (UE_flag==0) ? "eNB" : "UE",
	   1900+starttime.tm_year, starttime.tm_mon+1, starttime.tm_mday, starttime.tm_hour, starttime.tm_min, starttime.tm_sec);

  dumpfile_id = fopen(dumpfile_name,"w");
  if (dumpfile_id == NULL)
    {
      fprintf(stderr, "[EMOS] Error opening dumpfile %s\n",dumpfile_name);
      exit(EXIT_FAILURE);
    }


  printf("[EMOS] starting dump, channel_buffer_size=%d ...\n",channel_buffer_size);
  while (!oai_exit)
    {
      bytes = rtf_read_timed(fifo, fifo2file_ptr, channel_buffer_size,100);
      if (bytes==0)
	continue;

      /*
	if (UE_flag==0)
	printf("eNB: count %d, frame %d, read: %d bytes from the fifo\n",counter, ((fifo_dump_emos_eNB*)fifo2file_ptr)->frame_tx,bytes);
	else
	printf("UE: count %d, frame %d, read: %d bytes from the fifo\n",counter, ((fifo_dump_emos_UE*)fifo2file_ptr)->frame_rx,bytes);
      */

      fifo2file_ptr += channel_buffer_size;
      counter ++;

      if (counter == NO_ESTIMATES_DISK)
        {
          //reset stuff
          fifo2file_ptr = fifo2file_buffer;
          counter = 0;

          //flush buffer to disk
	  if (UE_flag==0)
	    printf("[EMOS] eNB: count %d, frame %d, flushing buffer to disk\n",
		   counter, ((fifo_dump_emos_eNB*)fifo2file_ptr)->frame_tx);
	  else
	    printf("[EMOS] UE: count %d, frame %d, flushing buffer to disk\n",
		   counter, ((fifo_dump_emos_UE*)fifo2file_ptr)->frame_rx);


          if (fwrite(fifo2file_buffer, sizeof(char), NO_ESTIMATES_DISK*channel_buffer_size, dumpfile_id) != NO_ESTIMATES_DISK*channel_buffer_size)
            {
              fprintf(stderr, "[EMOS] Error writing to dumpfile\n");
              exit(EXIT_FAILURE);
            }
	  if (gps_data)
	    {
	      if (gps_poll(gps_data) != 0) {
		printf("[EMOS] problem polling data from gps\n");
	      }
	      else {
		printf("[EMOS] lat %g, lon %g\n",gps_data->fix.latitude,gps_data->fix.longitude);
	      }
	      if (fwrite(&(gps_data->fix), sizeof(char), sizeof(struct gps_fix_t), dumpfile_id) != sizeof(struct gps_fix_t))
		{
		  printf("[EMOS] Error writing to dumpfile, stopping recording\n");
		  exit(EXIT_FAILURE);
		}
	    }
	  else
	    {
	      printf("[EMOS] WARNING: No GPS data available, storing dummy packet\n");
	      if (fwrite(&(dummy_gps_data), sizeof(char), sizeof(struct gps_fix_t), dumpfile_id) != sizeof(struct gps_fix_t))
		{
		  printf("[EMOS] Error writing to dumpfile, stopping recording\n");
		  exit(EXIT_FAILURE);
		}
	    } 
        }
    }
  
  free(fifo2file_buffer);
  fclose(dumpfile_id);
  close(fifo);
  
  pthread_exit((void*) arg);

}
#endif

#if defined(ENABLE_ITTI)
static void wait_system_ready (char *message, volatile int *start_flag)
{
  /* Wait for eNB application initialization to be complete (eNB registration to MME) */
  {
    static char *indicator[] = {".    ", "..   ", "...  ", ".... ", ".....",
                                " ....", "  ...", "   ..", "    .", "     "};
    int i = 0;

    while ((!oai_exit) && (*start_flag == 0)) {
      LOG_N(EMU, message, indicator[i]);
      i = (i + 1) % (sizeof(indicator) / sizeof(indicator[0]));
      usleep(200000);
    }
    LOG_D(EMU,"\n");
  }
}
#endif

#if defined(ENABLE_ITTI)
void *l2l1_task(void *arg)
{
  MessageDef *message_p = NULL;
  int         result;

  itti_set_task_real_time(TASK_L2L1);
  itti_mark_task_ready(TASK_L2L1);

  if (UE_flag == 0) {
    /* Wait for the initialize message */
    do {
      if (message_p != NULL) {
	result = itti_free (ITTI_MSG_ORIGIN_ID(message_p), message_p);
	AssertFatal (result == EXIT_SUCCESS, "Failed to free memory (%d)!\n", result);
      }
      itti_receive_msg (TASK_L2L1, &message_p);

      switch (ITTI_MSG_ID(message_p)) {
      case INITIALIZE_MESSAGE:
	/* Start eNB thread */
	LOG_D(EMU, "L2L1 TASK received %s\n", ITTI_MSG_NAME(message_p));
	start_eNB = 1;
	break;

      case TERMINATE_MESSAGE:
	oai_exit=1;
	itti_exit_task ();
	break;

      default:
	LOG_E(EMU, "Received unexpected message %s\n", ITTI_MSG_NAME(message_p));
	break;
      }
    } while (ITTI_MSG_ID(message_p) != INITIALIZE_MESSAGE);
    result = itti_free (ITTI_MSG_ORIGIN_ID(message_p), message_p);
    AssertFatal (result == EXIT_SUCCESS, "Failed to free memory (%d)!\n", result);
  }

  do {
    // Wait for a message
    itti_receive_msg (TASK_L2L1, &message_p);

    switch (ITTI_MSG_ID(message_p)) {
    case TERMINATE_MESSAGE:
      oai_exit=1;
      itti_exit_task ();
      break;

    case ACTIVATE_MESSAGE:
      start_UE = 1;
      break;

    case DEACTIVATE_MESSAGE:
      start_UE = 0;
      break;

    case MESSAGE_TEST:
      LOG_I(EMU, "Received %s\n", ITTI_MSG_NAME(message_p));
      break;

    default:
      LOG_E(EMU, "Received unexpected message %s\n", ITTI_MSG_NAME(message_p));
      break;
    }

    result = itti_free (ITTI_MSG_ORIGIN_ID(message_p), message_p);
    AssertFatal (result == EXIT_SUCCESS, "Failed to free memory (%d)!\n", result);
  } while(1);

  return NULL;
}
#endif


void do_OFDM_mod(int subframe,PHY_VARS_eNB *phy_vars_eNB) {

  unsigned int aa,slot_offset, slot_offset_F;
#ifndef USRP
  int dummy_tx_b[7680*4] __attribute__((aligned(16)));
#endif
  int i, tx_offset;
  int slot_sizeF = (phy_vars_eNB->lte_frame_parms.ofdm_symbol_size)*
    ((phy_vars_eNB->lte_frame_parms.Ncp==1) ? 6 : 7);

  slot_offset_F = (subframe<<1)*slot_sizeF;
    
  slot_offset = (subframe<<1)*
    (phy_vars_eNB->lte_frame_parms.samples_per_tti>>1);
  if ((subframe_select(&phy_vars_eNB->lte_frame_parms,subframe)==SF_DL)||
      ((subframe_select(&phy_vars_eNB->lte_frame_parms,subframe)==SF_S))) {
    //	  LOG_D(HW,"Frame %d: Generating slot %d\n",frame,next_slot);

#ifdef EXMIMO      
    for (aa=0; aa<phy_vars_eNB->lte_frame_parms.nb_antennas_tx; aa++) {
      if (phy_vars_eNB->lte_frame_parms.Ncp == EXTENDED){ 
	PHY_ofdm_mod(&phy_vars_eNB->lte_eNB_common_vars.txdataF[0][aa][slot_offset_F],
		     dummy_tx_b,
		     phy_vars_eNB->lte_frame_parms.log2_symbol_size,
		     6,
		     phy_vars_eNB->lte_frame_parms.nb_prefix_samples,
		     phy_vars_eNB->lte_frame_parms.twiddle_ifft,
		     phy_vars_eNB->lte_frame_parms.rev,
		     CYCLIC_PREFIX);
	PHY_ofdm_mod(&phy_vars_eNB->lte_eNB_common_vars.txdataF[0][aa][slot_offset_F+slot_sizeF],
		     dummy_tx_b+(phy_vars_eNB->lte_frame_parms.samples_per_tti>>1),
		     phy_vars_eNB->lte_frame_parms.log2_symbol_size,
		     6,
		     phy_vars_eNB->lte_frame_parms.nb_prefix_samples,
		     phy_vars_eNB->lte_frame_parms.twiddle_ifft,
		     phy_vars_eNB->lte_frame_parms.rev,
		     CYCLIC_PREFIX);
      }
      else {
	normal_prefix_mod(&phy_vars_eNB->lte_eNB_common_vars.txdataF[0][aa][slot_offset_F],
			  dummy_tx_b,
			  7,
			  &(phy_vars_eNB->lte_frame_parms));
	normal_prefix_mod(&phy_vars_eNB->lte_eNB_common_vars.txdataF[0][aa][slot_offset_F+slot_sizeF],
			  dummy_tx_b+(phy_vars_eNB->lte_frame_parms.samples_per_tti>>1),
			  7,
			  &(phy_vars_eNB->lte_frame_parms));
      }

      for (i=0; i<phy_vars_eNB->lte_frame_parms.samples_per_tti; i++) {
	tx_offset = (int)slot_offset+time_offset[aa]+i;
	if (tx_offset<0)
	  tx_offset += LTE_NUMBER_OF_SUBFRAMES_PER_FRAME*phy_vars_eNB->lte_frame_parms.samples_per_tti;
	if (tx_offset>=(LTE_NUMBER_OF_SUBFRAMES_PER_FRAME*phy_vars_eNB->lte_frame_parms.samples_per_tti))
	  tx_offset -= LTE_NUMBER_OF_SUBFRAMES_PER_FRAME*phy_vars_eNB->lte_frame_parms.samples_per_tti;
	((short*)&phy_vars_eNB->lte_eNB_common_vars.txdata[0][aa][tx_offset])[0]=
	  ((short*)dummy_tx_b)[2*i]<<4;
	((short*)&phy_vars_eNB->lte_eNB_common_vars.txdata[0][aa][tx_offset])[1]=
	  ((short*)dummy_tx_b)[2*i+1]<<4;
      }
    }
  
#else

    for (aa=0; aa<phy_vars_eNB->lte_frame_parms.nb_antennas_tx; aa++) {
      if (phy_vars_eNB->lte_frame_parms.Ncp == EXTENDED){ 
	PHY_ofdm_mod(&phy_vars_eNB->lte_eNB_common_vars.txdataF[0][aa][slot_offset_F],
		     &PHY_vars_eNB_g[0]->lte_eNB_common_vars.txdata[0][aa][slot_offset],
		     phy_vars_eNB->lte_frame_parms.log2_symbol_size,
		     6,
		     phy_vars_eNB->lte_frame_parms.nb_prefix_samples,
		     phy_vars_eNB->lte_frame_parms.twiddle_ifft,
		     phy_vars_eNB->lte_frame_parms.rev,
		     CYCLIC_PREFIX);
	PHY_ofdm_mod(&phy_vars_eNB->lte_eNB_common_vars.txdataF[0][aa][slot_offset_F+slot_sizeF],
		     &PHY_vars_eNB_g[0]->lte_eNB_common_vars.txdata[0][aa][slot_offset+(phy_vars_eNB->lte_frame_parms.samples_per_tti>>1)],
		     phy_vars_eNB->lte_frame_parms.log2_symbol_size,
		     6,
		     phy_vars_eNB->lte_frame_parms.nb_prefix_samples,
		     phy_vars_eNB->lte_frame_parms.twiddle_ifft,
		     phy_vars_eNB->lte_frame_parms.rev,
		     CYCLIC_PREFIX);
      }
      else {
	normal_prefix_mod(&phy_vars_eNB->lte_eNB_common_vars.txdataF[0][aa][slot_offset_F],
			  &PHY_vars_eNB_g[0]->lte_eNB_common_vars.txdata[aa][slot_offset],
			  7,
			  &(phy_vars_eNB->lte_frame_parms));
	normal_prefix_mod(&phy_vars_eNB->lte_eNB_common_vars.txdataF[0][aa][slot_offset_F+slot_sizeF],
			  &PHY_vars_eNB_g[0]->lte_eNB_common_vars.txdata[0][aa][slot_offset+(phy_vars_eNB->lte_frame_parms.samples_per_tti>>1)],
			  7,
			  &(phy_vars_eNB->lte_frame_parms));
      }

    }
#endif //EXMIMO
  }
}


int eNB_thread_tx_status[10];
static void * eNB_thread_tx(void *param) {

  //unsigned long cpuid;
  eNB_proc_t *proc = (eNB_proc_t*)param;
  int subframe_tx;
  //  RTIME time_in,time_out;
#ifdef RTAI
  RT_TASK *task;
  char task_name[32];
#endif

 

#if defined(ENABLE_ITTI)
  /* Wait for eNB application initialization to be complete (eNB registration to MME) */
  wait_system_ready ("Waiting for eNB application to be ready %s\r", &start_eNB);
#endif

#ifdef RTAI
  sprintf(task_name,"eNB_proc_TX %d",proc->subframe);
  task = rt_task_init_schmod(nam2num(task_name), 0, 0, 0, SCHED_FIFO, 0xF);

  if (task==NULL) {
    LOG_E(PHY,"[SCHED][eNB] Problem starting eNB_proc_TX thread_index %d (%s)!!!!\n",proc->subframe,task_name);
    return 0;
  }
  else {
    LOG_I(PHY,"[SCHED][eNB] eNB TX thread %d started with id %p\n",
	  proc->subframe,
	  task);
  }
#else
  LOG_I(PHY,"[SCHED][eNB] eNB TX thread %d started on CPU %d\n",
	proc->subframe,sched_getcpu());
#endif

  mlockall(MCL_CURRENT | MCL_FUTURE);

  //rt_set_runnable_on_cpuid(task,1);
  //cpuid = rtai_cpuid();

#ifdef HARD_RT
  rt_make_hard_real_time();
#endif


  subframe_tx = (proc->subframe+1)%10;
  
  while (!oai_exit){
    
    vcd_signal_dumper_dump_function_by_name(VCD_SIGNAL_DUMPER_FUNCTIONS_eNB_PROC_TX0+(2*proc->subframe),0);
    
    
    //    LOG_I(PHY,"Locking mutex for eNB proc %d (IC %d,mutex %p)\n",proc->subframe,proc->instance_cnt,&proc->mutex);
    if (pthread_mutex_lock(&proc->mutex_tx) != 0) {
      LOG_E(PHY,"[SCHED][eNB] error locking mutex for eNB TX proc %d\n",proc->subframe);
    }
    else {
      
      while (proc->instance_cnt_tx < 0) {
	//	LOG_I(PHY,"Waiting and unlocking mutex for eNB proc %d (IC %d,lock %d)\n",proc->subframe,proc->instance_cnt,pthread_mutex_trylock(&proc->mutex));
	
	pthread_cond_wait(&proc->cond_tx,&proc->mutex_tx);
      }
      //      LOG_I(PHY,"Waking up and unlocking mutex for eNB proc %d\n",proc->subframe);
      if (pthread_mutex_unlock(&proc->mutex_tx) != 0) {	
	LOG_E(PHY,"[SCHED][eNB] error unlocking mutex for eNB TX proc %d\n",proc->subframe);
      }
    }
    vcd_signal_dumper_dump_function_by_name(VCD_SIGNAL_DUMPER_FUNCTIONS_eNB_PROC_TX0+(2*proc->subframe),1);    
    vcd_signal_dumper_dump_variable_by_name(VCD_SIGNAL_DUMPER_VARIABLES_FRAME_NUMBER_ENB, proc->frame_tx);
    vcd_signal_dumper_dump_variable_by_name(VCD_SIGNAL_DUMPER_VARIABLES_SLOT_NUMBER_ENB, proc->subframe*2);
    
    if (oai_exit) break;
    
    if ((((PHY_vars_eNB_g[0]->lte_frame_parms.frame_type == TDD)&&(subframe_select(&PHY_vars_eNB_g[0]->lte_frame_parms,subframe_tx)==SF_DL))||
	 (PHY_vars_eNB_g[0]->lte_frame_parms.frame_type == FDD))) {
      phy_procedures_eNB_TX(subframe_tx,PHY_vars_eNB_g[0],0,no_relay,NULL);
    }
    if ((subframe_select(&PHY_vars_eNB_g[0]->lte_frame_parms,subframe_tx)==SF_S)) {
      phy_procedures_eNB_TX(subframe_tx,PHY_vars_eNB_g[0],0,no_relay,NULL);
    }
    do_OFDM_mod(subframe_tx,PHY_vars_eNB_g[0]);  
    
    if (pthread_mutex_lock(&proc->mutex_tx) != 0) {
      printf("[openair][SCHED][eNB] error locking mutex for eNB TX proc %d\n",proc->subframe);
    }
    else {
      proc->instance_cnt_tx--;
      
      if (pthread_mutex_unlock(&proc->mutex_tx) != 0) {	
	printf("[openair][SCHED][eNB] error unlocking mutex for eNB TX proc %d\n",proc->subframe);
      }
    }
    
    proc->frame_tx++;
    if (proc->frame_tx==1024)
      proc->frame_tx=0;
  }    
  vcd_signal_dumper_dump_function_by_name(VCD_SIGNAL_DUMPER_FUNCTIONS_eNB_PROC_TX0+(2*proc->subframe),0);        
#ifdef HARD_RT
  rt_make_soft_real_time();
#endif
  
#ifdef DEBUG_THREADS
  printf("Exiting eNB thread TX %d\n",proc->subframe);
#endif
  // clean task
#ifdef RTAI
  rt_task_delete(task);
#else
  eNB_thread_tx_status[proc->subframe]=0;
  pthread_exit(&eNB_thread_tx_status[proc->subframe]);
#endif
  
#ifdef DEBUG_THREADS
  printf("Exiting eNB TX thread %d\n",proc->subframe);
#endif
}

int eNB_thread_rx_status[10];
static void * eNB_thread_rx(void *param) {

  //unsigned long cpuid;
  eNB_proc_t *proc = (eNB_proc_t*)param;
  int i;
  int subframe_rx;
  //  RTIME time_in,time_out;
#ifdef RTAI
  RT_TASK *task;
  char task_name[32];
#endif

#if defined(ENABLE_ITTI)
  /* Wait for eNB application initialization to be complete (eNB registration to MME) */
  wait_system_ready ("Waiting for eNB application to be ready %s\r", &start_eNB);
#endif

#ifdef RTAI
  sprintf(task_name,"eNB_proc_RX %d",proc->subframe);
  task = rt_task_init_schmod(nam2num(task_name), 0, 0, 0, SCHED_FIFO, 0xF);

  if (task==NULL) {
    LOG_E(PHY,"[SCHED][eNB] Problem starting eNB_proc_RX thread_index %d (%s)!!!!\n",proc->subframe,task_name);
    return 0;
  }
  else {
    LOG_I(PHY,"[SCHED][eNB] eNB RX thread %d started with id %p\n", /*  on CPU %d*/
	  proc->subframe,
	  task); /*,rtai_cpuid()*/
  }
#else
  LOG_I(PHY,"[SCHED][eNB] eNB RX thread %d started on CPU %d\n",
	proc->subframe,sched_getcpu());
#endif

  mlockall(MCL_CURRENT | MCL_FUTURE);

  //rt_set_runnable_on_cpuid(task,1);
  //cpuid = rtai_cpuid();

#ifdef HARD_RT
  rt_make_hard_real_time();
#endif


  subframe_rx = (proc->subframe+9)%10;

  while (!oai_exit){

    vcd_signal_dumper_dump_function_by_name(VCD_SIGNAL_DUMPER_FUNCTIONS_eNB_PROC_RX0+(2*proc->subframe),0);


    //    LOG_I(PHY,"Locking mutex for eNB proc %d (IC %d,mutex %p)\n",proc->subframe,proc->instance_cnt,&proc->mutex);
    if (pthread_mutex_lock(&proc->mutex_rx) != 0) {
      LOG_E(PHY,"[SCHED][eNB] error locking mutex for eNB RX proc %d\n",proc->subframe);
    }
    else {
        
      while (proc->instance_cnt_rx < 0) {
	//	LOG_I(PHY,"Waiting and unlocking mutex for eNB proc %d (IC %d,lock %d)\n",proc->subframe,proc->instance_cnt,pthread_mutex_trylock(&proc->mutex));

	pthread_cond_wait(&proc->cond_rx,&proc->mutex_rx);
      }
      //      LOG_I(PHY,"Waking up and unlocking mutex for eNB proc %d\n",proc->subframe);
      if (pthread_mutex_unlock(&proc->mutex_rx) != 0) {	
	LOG_E(PHY,"[SCHED][eNB] error unlocking mutex for eNB RX proc %d\n",proc->subframe);
      }
    }
    vcd_signal_dumper_dump_function_by_name(VCD_SIGNAL_DUMPER_FUNCTIONS_eNB_PROC_RX0+(2*proc->subframe),1);    

    if (oai_exit) break;
    
    if ((((PHY_vars_eNB_g[0]->lte_frame_parms.frame_type == TDD )&&(subframe_select(&PHY_vars_eNB_g[0]->lte_frame_parms,subframe_rx)==SF_UL)) ||
	 (PHY_vars_eNB_g[0]->lte_frame_parms.frame_type == FDD))){
      phy_procedures_eNB_RX(subframe_rx,PHY_vars_eNB_g[0],0,no_relay);
    }
    if ((subframe_select(&PHY_vars_eNB_g[0]->lte_frame_parms,subframe_rx)==SF_S)){
      phy_procedures_eNB_S_RX(subframe_rx,PHY_vars_eNB_g[0],0,no_relay);
    }
      
    if (pthread_mutex_lock(&proc->mutex_rx) != 0) {
      printf("[openair][SCHED][eNB] error locking mutex for eNB RX proc %d\n",proc->subframe);
    }
    else {
      proc->instance_cnt_rx--;
      
      if (pthread_mutex_unlock(&proc->mutex_rx) != 0) {	
	printf("[openair][SCHED][eNB] error unlocking mutex for eNB RX proc %d\n",proc->subframe);
      }
    }

    proc->frame_rx++;
    if (proc->frame_rx==1024)
      proc->frame_rx=0;
    
  }
  vcd_signal_dumper_dump_function_by_name(VCD_SIGNAL_DUMPER_FUNCTIONS_eNB_PROC_RX0+(2*proc->subframe),0);        
#ifdef HARD_RT
  rt_make_soft_real_time();
#endif

#ifdef DEBUG_THREADS
  printf("Exiting eNB thread RX %d\n",proc->subframe);
#endif
  // clean task
#ifdef RTAI
  rt_task_delete(task);
#else
  eNB_thread_rx_status[proc->subframe]=0;
  pthread_exit(&eNB_thread_rx_status[proc->subframe]);
#endif

#ifdef DEBUG_THREADS
  printf("Exiting eNB RX thread %d\n",proc->subframe);
#endif
}



void init_eNB_proc() {

  int i;



  for (i=0;i<10;i++) {
    pthread_attr_init (&attr_eNB_proc_tx[i]);
    pthread_attr_setstacksize(&attr_eNB_proc_tx[i],OPENAIR_THREAD_STACK_SIZE);
    //attr_dlsch_threads.priority = 1;
    sched_param_eNB_proc_tx[i].sched_priority = sched_get_priority_max(SCHED_FIFO)-1; //OPENAIR_THREAD_PRIORITY;
    pthread_attr_setschedparam  (&attr_eNB_proc_tx[i], &sched_param_eNB_proc_tx);
    pthread_attr_setschedpolicy (&attr_eNB_proc_tx[i], SCHED_FIFO);

    pthread_attr_init (&attr_eNB_proc_rx[i]);
    pthread_attr_setstacksize(&attr_eNB_proc_rx[i],OPENAIR_THREAD_STACK_SIZE);
    //attr_dlsch_threads.priority = 1;
    sched_param_eNB_proc_rx[i].sched_priority = sched_get_priority_max(SCHED_FIFO)-1; //OPENAIR_THREAD_PRIORITY;
    pthread_attr_setschedparam  (&attr_eNB_proc_rx[i], &sched_param_eNB_proc_rx);
    pthread_attr_setschedpolicy (&attr_eNB_proc_rx[i], SCHED_FIFO);

    PHY_vars_eNB_g[0]->proc[i].instance_cnt_tx=-1;
    PHY_vars_eNB_g[0]->proc[i].instance_cnt_rx=-1;
    PHY_vars_eNB_g[0]->proc[i].subframe=i;
    pthread_mutex_init(&PHY_vars_eNB_g[0]->proc[i].mutex_tx,NULL);
    pthread_mutex_init(&PHY_vars_eNB_g[0]->proc[i].mutex_rx,NULL);
    pthread_cond_init(&PHY_vars_eNB_g[0]->proc[i].cond_tx,NULL);
    pthread_cond_init(&PHY_vars_eNB_g[0]->proc[i].cond_rx,NULL);
    pthread_create(&PHY_vars_eNB_g[0]->proc[i].pthread_tx,NULL,eNB_thread_tx,(void*)&PHY_vars_eNB_g[0]->proc[i]);
    pthread_create(&PHY_vars_eNB_g[0]->proc[i].pthread_rx,NULL,eNB_thread_rx,(void*)&PHY_vars_eNB_g[0]->proc[i]);
  }
}

void kill_eNB_proc() {

  int i;
  int *status_tx,*status_rx;

  for (i=0;i<10;i++) {

#ifdef DEBUG_THREADS
    printf("Killing TX thread %d\n",i);
#endif
    PHY_vars_eNB_g[0]->proc[i].instance_cnt_tx=0; 
    pthread_cond_signal(&PHY_vars_eNB_g[0]->proc[i].cond_tx);
#ifdef DEBUG_THREADS
    printf("Joining eNB TX thread %d...",i);
#endif
    pthread_join(PHY_vars_eNB_g[0]->proc[i].pthread_tx,(void**)status_tx);
#ifdef DEBUG_THREADS
    if (status_tx) printf("status %d...",*status_tx);
#endif
#ifdef DEBUG_THREADS
    printf("Killing RX thread %d\n",i);
#endif
    PHY_vars_eNB_g[0]->proc[i].instance_cnt_rx=0; 
    pthread_cond_signal(&PHY_vars_eNB_g[0]->proc[i].cond_rx);
#ifdef DEBUG_THREADS
    printf("Joining eNB RX thread %d...",i);
#endif
    pthread_join(PHY_vars_eNB_g[0]->proc[i].pthread_rx,(void**)status_rx);
#ifdef DEBUG_THREADS 
    if (status_rx) printf("status %d...",*status_rx);
#endif
    pthread_mutex_destroy(&PHY_vars_eNB_g[0]->proc[i].mutex_tx);
    pthread_mutex_destroy(&PHY_vars_eNB_g[0]->proc[i].mutex_rx);
    pthread_cond_destroy(&PHY_vars_eNB_g[0]->proc[i].cond_tx);
    pthread_cond_destroy(&PHY_vars_eNB_g[0]->proc[i].cond_rx);
  }
}




  
/* This is the main eNB thread. */
int eNB_thread_status;


static void *eNB_thread(void *arg)
{
#ifdef RTAI
  RT_TASK *task;
#endif
  unsigned char slot=0;//,last_slot, next_slot;
  int frame=0;

  RTIME time_in, time_diff;

  int sf;
#ifndef USRP
  volatile unsigned int *DAQ_MBOX = openair0_daq_cnt();
  int mbox_target=0,mbox_current=0;
  int hw_slot,delay_cnt;
  int diff;
  int i;
  int ret;

#else
  int rx_cnt = 0;
  int tx_cnt = tx_delay;
  //  int tx_offset;

  hw_subframe = 0;
#endif
#if defined(ENABLE_ITTI)
  /* Wait for eNB application initialization to be complete (eNB registration to MME) */
  wait_system_ready ("Waiting for eNB application to be ready %s\r", &start_eNB);
#endif

#ifdef RTAI
  task = rt_task_init_schmod(nam2num("TASK0"), 0, 0, 0, SCHED_FIFO, 0xF);
#endif

  if (!oai_exit) {
#ifdef RTAI
    LOG_D(HW,"[SCHED][eNB] Started eNB thread (id %p)\n",task/*" on CPU %d",rtai_cpuid()*/);
#else
    LOG_I(HW,"[SCHED][eNB] Started eNB thread on CPU %d\n",
	  sched_getcpu());
#endif

#ifdef HARD_RT
    rt_make_hard_real_time();
#endif

    mlockall(MCL_CURRENT | MCL_FUTURE);

    timing_info.time_min = 100000000ULL;
    timing_info.time_max = 0;
    timing_info.time_avg = 0;
    timing_info.n_samples = 0;

    while (!oai_exit) {

#ifndef USRP
      hw_slot = (((((volatile unsigned int *)DAQ_MBOX)[0]+1)%150)<<1)/15;
      //        LOG_D(HW,"eNB frame %d, time %llu: slot %d, hw_slot %d (mbox %d)\n",frame,rt_get_time_ns(),slot,hw_slot,((unsigned int *)DAQ_MBOX)[0]);
      //this is the mbox counter where we should be
      //mbox_target = ((((slot+1)%20)*15+1)>>1)%150;
      mbox_target = mbox_bounds[slot];
      //this is the mbox counter where we are
      mbox_current = ((volatile unsigned int *)DAQ_MBOX)[0];
      //this is the time we need to sleep in order to synchronize with the hw (in multiples of DAQ_PERIOD)
      if ((mbox_current>=135) && (mbox_target<15)) //handle the frame wrap-arround
	diff = 150-mbox_current+mbox_target;
      else if ((mbox_current<15) && (mbox_target>=135))
	diff = -150+mbox_target-mbox_current;
      else
	diff = mbox_target - mbox_current;
      
      if (((slot%2==0) && (diff < (-14))) || ((slot%2==1) && (diff < (-7)))) {
	// at the eNB, even slots have double as much time since most of the processing is done here and almost nothing in odd slots
	LOG_D(HW,"eNB Frame %d, time %llu: missed slot, proceeding with next one (slot %d, hw_slot %d, diff %d)\n",frame, rt_get_time_ns(), slot, hw_slot, diff);
	slot++;
	if (frame > 0) {
	  exit_fun("[HW][eNB] missed slot");
	}
	if (slot==20){
	  slot=0;
	  frame++;
	}
	continue;
      }
      if (diff>8)
	LOG_D(HW,"eNB Frame %d, time %llu: skipped slot, waiting for hw to catch up (slot %d, hw_slot %d, mbox_current %d, mbox_target %d, diff %d)\n",frame, rt_get_time_ns(), slot, hw_slot, mbox_current, mbox_target, diff);
      
      vcd_signal_dumper_dump_variable_by_name(VCD_SIGNAL_DUMPER_VARIABLES_DAQ_MBOX, *DAQ_MBOX);
      vcd_signal_dumper_dump_variable_by_name(VCD_SIGNAL_DUMPER_VARIABLES_DIFF, diff);
      
      delay_cnt = 0;
      while ((diff>0) && (!oai_exit)) {
	time_in = rt_get_time_ns();
	//LOG_D(HW,"eNB Frame %d delaycnt %d : hw_slot %d (%d), slot %d, (slot+1)*15=%d, diff %d, time %llu\n",frame,delay_cnt,hw_slot,((unsigned int *)DAQ_MBOX)[0],slot,(((slot+1)*15)>>1),diff,time_in);
	//LOG_D(HW,"eNB Frame %d, time %llu: sleeping for %llu (slot %d, hw_slot %d, diff %d, mbox %d, delay_cnt %d)\n", frame, time_in, diff*DAQ_PERIOD,slot,hw_slot,diff,((volatile unsigned int *)DAQ_MBOX)[0],delay_cnt);
	vcd_signal_dumper_dump_function_by_name(VCD_SIGNAL_DUMPER_FUNCTIONS_RT_SLEEP,1);
	ret = rt_sleep_ns(diff*DAQ_PERIOD);
	vcd_signal_dumper_dump_function_by_name(VCD_SIGNAL_DUMPER_FUNCTIONS_RT_SLEEP,0);
	vcd_signal_dumper_dump_variable_by_name(VCD_SIGNAL_DUMPER_VARIABLES_DAQ_MBOX, *DAQ_MBOX);
	if (ret)
	  LOG_D(HW,"eNB Frame %d, time %llu: rt_sleep_ns returned %d\n",frame, time_in);
	hw_slot = (((((volatile unsigned int *)DAQ_MBOX)[0]+1)%150)<<1)/15;
	//LOG_D(HW,"eNB Frame %d : hw_slot %d, time %llu\n",frame,hw_slot,rt_get_time_ns());
	delay_cnt++;
	if (delay_cnt == 10) {
	  LOG_D(HW,"eNB Frame %d: HW stopped ... \n",frame);
	  exit_fun("[HW][eNB] HW stopped");
	}
	mbox_current = ((volatile unsigned int *)DAQ_MBOX)[0];
	if ((mbox_current>=135) && (mbox_target<15)) //handle the frame wrap-arround
	  diff = 150-mbox_current+mbox_target;
	else if ((mbox_current<15) && (mbox_target>=135))
	  diff = -150+mbox_target-mbox_current;
	else
	  diff = mbox_target - mbox_current;
      }

#else  // USRP
      while (rx_cnt < sf_bounds[hw_subframe]) {
	openair0.trx_read_func(&openair0, &timestamp, &rxdata[rx_cnt*samples_per_packets], samples_per_packets);

	openair0.trx_write_func(&openair0, (timestamp+samples_per_packets*tx_delay-tx_forward_nsamps), &txdata[tx_cnt*samples_per_packets], samples_per_packets, 1);

	rx_cnt++;
	tx_cnt++;
      }

#ifndef RTAI
      //pthread_mutex_lock(&tti_mutex);
#endif
      hw_subframe++;
      slot+=2;
      if(hw_subframe==10)
        hw_subframe = 0;

#endif // USRP
     
      if (oai_exit) break;

      if (frame>5)  {

	timing_info.time_last = timing_info.time_now;
	timing_info.time_now = rt_get_time_ns();
	
	if (timing_info.n_samples>0) {
	  time_diff = timing_info.time_now - timing_info.time_last;
	  if (time_diff < timing_info.time_min)
	    timing_info.time_min = time_diff;
	  if (time_diff > timing_info.time_max)
	    timing_info.time_max = time_diff;
	  timing_info.time_avg += time_diff;
	}
	
	timing_info.n_samples++;
	/*
	  if ((timing_info.n_samples%2000)==0) {
	  LOG_D(HW,"frame %d (%d), slot %d, hw_slot %d: diff=%llu, min=%llu, max=%llu, avg=%llu (n_samples %d)\n",
	  frame, PHY_vars_eNB_g[0]->frame, slot, hw_slot,time_diff,
	  timing_info.time_min,timing_info.time_max,timing_info.time_avg/timing_info.n_samples,timing_info.n_samples);
	  timing_info.n_samples = 0;
	  timing_info.time_avg = 0;
	  }
	*/
	//}
	
	if (multi_thread == 0) {
	  if ((slot&1) == 0) {
	    LOG_I(PHY,"[eNB] Single thread slot %d\n",slot);
	    phy_procedures_eNB_lte ((2+(slot>>1))%10, PHY_vars_eNB_g[0], 0, no_relay,NULL);
	    do_OFDM_mod((2+(slot>>1))%10,PHY_vars_eNB_g[0]);
	  }
	}
	else { // multi-thread > 0
	  if ((slot&1) == 0) {
	    sf = ((slot>>1)+1)%10;
	    //		    LOG_I(PHY,"[eNB] Multithread slot %d (IC %d)\n",slot,PHY_vars_eNB_g[0]->proc[sf].instance_cnt);
	    
	    if (pthread_mutex_lock(&PHY_vars_eNB_g[0]->proc[sf].mutex_tx) != 0) {
	      LOG_E(PHY,"[eNB] ERROR pthread_mutex_lock for eNB TX thread %d (IC %d)\n",sf,PHY_vars_eNB_g[0]->proc[sf].instance_cnt_tx);   
	    }
	    else {
	      //		      LOG_I(PHY,"[eNB] Waking up eNB process %d (IC %d)\n",sf,PHY_vars_eNB_g[0]->proc[sf].instance_cnt); 
	      PHY_vars_eNB_g[0]->proc[sf].instance_cnt_tx++;
	      pthread_mutex_unlock(&PHY_vars_eNB_g[0]->proc[sf].mutex_tx);
	      if (PHY_vars_eNB_g[0]->proc[sf].instance_cnt_tx == 0) {
		if (pthread_cond_signal(&PHY_vars_eNB_g[0]->proc[sf].cond_tx) != 0) {
		  LOG_E(PHY,"[eNB] ERROR pthread_cond_signal for eNB TX thread %d\n",sf);
		}
	      }
	      else {
		LOG_W(PHY,"[eNB] Frame %d, eNB TX thread %d busy!!\n",PHY_vars_eNB_g[0]->proc[sf].frame_tx,sf);
	      }
	    }
	    
	    if (pthread_mutex_lock(&PHY_vars_eNB_g[0]->proc[sf].mutex_rx) != 0) {
	      LOG_E(PHY,"[eNB] ERROR pthread_mutex_lock for eNB RX thread %d (IC %d)\n",sf,PHY_vars_eNB_g[0]->proc[sf].instance_cnt_rx);   
	    }
	    else {
	      //		      LOG_I(PHY,"[eNB] Waking up eNB process %d (IC %d)\n",sf,PHY_vars_eNB_g[0]->proc[sf].instance_cnt); 
	      PHY_vars_eNB_g[0]->proc[sf].instance_cnt_rx++;
	      pthread_mutex_unlock(&PHY_vars_eNB_g[0]->proc[sf].mutex_rx);
	      if (PHY_vars_eNB_g[0]->proc[sf].instance_cnt_rx == 0) {
		if (pthread_cond_signal(&PHY_vars_eNB_g[0]->proc[sf].cond_rx) != 0) {
		  LOG_E(PHY,"[eNB] ERROR pthread_cond_signal for eNB RX thread %d\n",sf);
		}
	      }
	      else {
		LOG_W(PHY,"[eNB] Frame %d, eNB RX thread %d busy!!\n",PHY_vars_eNB_g[0]->proc[sf].frame_rx,sf);
	      }
	    }
	    
	  }
	}
      }
#ifndef USRP
      slot++;
#else
      if(rx_cnt == max_cnt) {
	rx_cnt = 0;
      } 
#endif     

      if (slot==20) {
	slot=0;
	frame++;
      }
#if defined(ENABLE_ITTI)
      itti_update_lte_time(frame, slot);
#endif
    }
  }
#ifdef DEBUG_THREADS
  printf("eNB_thread: finished, ran %d times.\n",frame);
#endif
  
#ifdef HARD_RT
  rt_make_soft_real_time();
#endif


#ifdef DEBUG_THREADS
  printf("Exiting eNB_thread ...");
#endif
  // clean task
#ifdef RTAI
  rt_task_delete(task);
#else
  eNB_thread_status = 0;
  pthread_exit(&eNB_thread_status);
#endif
#ifdef DEBUG_THREADS
  printf("eNB_thread deleted. returning\n");
#endif
  return 0;
}



#ifndef USRP
/* This is the main UE thread. Initially it is doing a periodic get_frame. One synchronized it gets woken up by the kernel driver using the RTAI message mechanism (rt_send and rt_receive). */
static void *UE_thread(void *arg) {
#ifdef RTAI
  RT_TASK *task;
#endif
  // RTIME in, out, diff;
  int slot=0,frame=0,hw_slot,last_slot, next_slot;
  // unsigned int aa;
  static int is_synchronized = 0;
  int delay_cnt;
  RTIME time_in;
  int hw_slot_offset=0,rx_offset_mbox=0,mbox_target=0,mbox_current=0;
  int diff2;
  int i, ret;
  volatile unsigned int *DAQ_MBOX = openair0_daq_cnt();
#ifndef USRP
  exmimo_config_t *p_exmimo_config = openair0_exmimo_pci[card].exmimo_config_ptr;;
#endif


#if defined(ENABLE_ITTI) && defined(ENABLE_USE_MME)
  /* Wait for NAS UE to start cell selection */
  wait_system_ready ("Waiting for UE to be activated by UserProcess %s\r", &start_UE);
#endif

#ifdef RTAI
  task = rt_task_init_schmod(nam2num("TASK0"), 0, 0, 0, SCHED_FIFO, 0xF);
  LOG_D(HW,"Started UE thread (id %p)\n",task);
#endif

#ifdef HARD_RT
  rt_make_hard_real_time();
#endif

  mlockall(MCL_CURRENT | MCL_FUTURE);

  openair_daq_vars.freq_offset = 0; //-7500;
  /*
    if (mode == rx_calib_ue) {
    openair_daq_vars.freq_offset = -7500;
    for (i=0; i<4; i++) {
    p_exmimo_config->rf.rf_freq_rx[i] = p_exmimo_config->rf.rf_freq_rx[i]+openair_daq_vars.freq_offset;
    p_exmimo_config->rf.rf_freq_tx[i] = p_exmimo_config->rf.rf_freq_rx[i]+openair_daq_vars.freq_offset;
    }
    openair0_dump_config(card);
    }
  */
  while (!oai_exit)  {
    hw_slot = (((((volatile unsigned int *)DAQ_MBOX)[0]+1)%150)<<1)/15; //the slot the hw is about to store
      
    if (is_synchronized) {
      //this is the mbox counter that indicates the start of the frame
      rx_offset_mbox = (PHY_vars_UE_g[0]->rx_offset * 150) / (10*PHY_vars_UE_g[0]->lte_frame_parms.samples_per_tti);
      //this is the mbox counter where we should be
      mbox_target = (((((slot+1)%20)*15+1)>>1) + rx_offset_mbox + 1)%150;
      // round up to the next multiple of two (mbox counter from express MIMO gives only even numbers)
      mbox_target = ((mbox_target+1)-((mbox_target-1)%2))%150;
      //this is the mbox counter where we are
      mbox_current = ((volatile unsigned int *)DAQ_MBOX)[0];
      //this is the time we need to sleep in order to synchronize with the hw (in multiples of DAQ_PERIOD)
      if ((mbox_current>=120) && (mbox_target<30)) //handle the frame wrap-arround
	diff2 = 150-mbox_current+mbox_target;
      else if ((mbox_current<30) && (mbox_target>=120))
	diff2 = -150+mbox_target-mbox_current;
      else
	diff2 = mbox_target - mbox_current;
	
      if (diff2 <(-7)) {
	LOG_D(HW,"UE Frame %d: missed slot, proceeding with next one (slot %d, hw_slot %d, diff %d)\n",frame, slot, hw_slot, diff2);
	if (frame>0)
	  exit_fun("[HW][UE] missed slot");
	slot++;
	if (slot==20) {
	  slot=0;
	  frame++;
	}
	continue;
      }
      if (diff2>8)
	LOG_D(HW,"UE Frame %d: skipped slot, waiting for hw to catch up (slot %d, hw_slot %d, mbox_current %d, mbox_target %d, diff %d)\n",frame, slot, hw_slot, mbox_current, mbox_target, diff2);
	
      /*
	if (frame%100==0)
	LOG_D(HW,"frame %d (%d), slot %d, hw_slot %d, rx_offset_mbox %d, mbox_target %d, mbox_current %d, diff %d\n",frame, PHY_vars_UE_g[0]->frame, slot,hw_slot,rx_offset_mbox,mbox_target,mbox_current,diff2);
      */
	
      vcd_signal_dumper_dump_variable_by_name(VCD_SIGNAL_DUMPER_VARIABLES_DAQ_MBOX, *DAQ_MBOX);
      vcd_signal_dumper_dump_variable_by_name(VCD_SIGNAL_DUMPER_VARIABLES_DIFF, diff2);
	
      delay_cnt = 0;
      while ((diff2>0) && (!oai_exit) && (is_synchronized) )  {
	time_in = rt_get_time_ns();
	//LOG_D(HW,"eNB Frame %d delaycnt %d : hw_slot %d (%d), slot %d (%d), diff %d, time %llu\n",frame,delay_cnt,hw_slot,((volatile unsigned int *)DAQ_MBOX)[0],slot,mbox_target,diff2,time_in);
	vcd_signal_dumper_dump_variable_by_name(VCD_SIGNAL_DUMPER_VARIABLES_DAQ_MBOX, *DAQ_MBOX);
	vcd_signal_dumper_dump_function_by_name(VCD_SIGNAL_DUMPER_FUNCTIONS_RT_SLEEP,1);
	ret = rt_sleep_ns(diff2*DAQ_PERIOD);
	vcd_signal_dumper_dump_function_by_name(VCD_SIGNAL_DUMPER_FUNCTIONS_RT_SLEEP,0);
	vcd_signal_dumper_dump_variable_by_name(VCD_SIGNAL_DUMPER_VARIABLES_DAQ_MBOX, *DAQ_MBOX);
	if (ret)
	  LOG_D(HW,"eNB Frame %d, time %llu: rt_sleep_ns returned %d\n",frame, time_in);
	  
	hw_slot = (((((volatile unsigned int *)DAQ_MBOX)[0]+1)%150)<<1)/15;
	//LOG_D(HW,"eNB Frame %d : hw_slot %d, time %llu\n",frame,hw_slot,rt_get_time_ns());
	delay_cnt++;
	if (delay_cnt == 30)  {
	  LOG_D(HW,"UE frame %d: HW stopped ... \n",frame);
	  exit_fun("[HW][UE] HW stopped");
	}
	mbox_current = ((volatile unsigned int *)DAQ_MBOX)[0];
	if ((mbox_current>=135) && (mbox_target<15)) //handle the frame wrap-arround
	  diff2 = 150-mbox_current+mbox_target;
	else if ((mbox_current<15) && (mbox_target>=135))
	  diff2 = -150+mbox_target-mbox_current;
	else
	  diff2 = mbox_target - mbox_current;
	  
	vcd_signal_dumper_dump_variable_by_name(VCD_SIGNAL_DUMPER_VARIABLES_DAQ_MBOX, *DAQ_MBOX);
	vcd_signal_dumper_dump_variable_by_name(VCD_SIGNAL_DUMPER_VARIABLES_DIFF, diff2);
      }
	
    }
      
    last_slot = (slot)%LTE_SLOTS_PER_FRAME;
    if (last_slot <0)
      last_slot+=LTE_SLOTS_PER_FRAME;
    next_slot = (slot+3)%LTE_SLOTS_PER_FRAME;
      
    if (is_synchronized)  {
      phy_procedures_UE_lte (last_slot, next_slot, PHY_vars_UE_g[0], 0, 0,mode,0,NULL);
	
    }
    else {  // we are not yet synchronized
      hw_slot_offset = 0;
	
      slot = 0;
      openair0_get_frame(card);
      //          LOG_D(HW,"after get_frame\n");
      //          rt_sleep_ns(FRAME_PERIOD);
      //          LOG_D(HW,"after sleep\n");
	
      if (initial_sync(PHY_vars_UE_g[0],mode)==0) {
	/*
	  lte_adjust_synch(&PHY_vars_UE_g[0]->lte_frame_parms,
	  PHY_vars_UE_g[0],
	  0,
	  1,
	  16384);
	*/
	//for better visualization afterwards
	/*
	  for (aa=0; aa<PHY_vars_UE_g[0]->lte_frame_parms.nb_antennas_rx; aa++)
	  memset(PHY_vars_UE_g[0]->lte_ue_common_vars.rxdata[aa],0,
	  PHY_vars_UE_g[0]->lte_frame_parms.samples_per_tti*LTE_NUMBER_OF_SUBFRAMES_PER_FRAME*sizeof(int));
	*/
	if (mode == rx_calib_ue) {
	  exit_fun("[HW][UE] UE in RX calibration mode");
	}
	else {
	  is_synchronized = 1;
	  //start the DMA transfers
	  //LOG_D(HW,"Before openair0_start_rt_acquisition \n");
	  openair0_start_rt_acquisition(card);
	    
	  hw_slot_offset = (PHY_vars_UE_g[0]->rx_offset<<1) / PHY_vars_UE_g[0]->lte_frame_parms.samples_per_tti;
	  LOG_D(HW,"Got synch: hw_slot_offset %d\n",hw_slot_offset);
	}
      }
      else {
	if (openair_daq_vars.freq_offset >= 0) {
	  openair_daq_vars.freq_offset += 100;
	  openair_daq_vars.freq_offset *= -1;
	}
	else {
	  openair_daq_vars.freq_offset *= -1;
	}
	if (abs(openair_daq_vars.freq_offset) > 7500) {
	  LOG_I(PHY,"[initial_sync] No cell synchronization found, abondoning\n");
	  mac_xface->macphy_exit("No cell synchronization found, abondoning");
	}
	else {
	  LOG_I(PHY,"[initial_sync] trying carrier off %d Hz\n",openair_daq_vars.freq_offset);
#ifndef USRP
	  for (i=0; i<4; i++) {
	    if (p_exmimo_config->rf.rf_freq_rx[i])
	      p_exmimo_config->rf.rf_freq_rx[i] = carrier_freq[i]+openair_daq_vars.freq_offset;
	    if (p_exmimo_config->rf.rf_freq_tx[i])
	      p_exmimo_config->rf.rf_freq_tx[i] = carrier_freq[i]+openair_daq_vars.freq_offset;
	  }
#endif
	  openair0_dump_config(card);
	  rt_sleep_ns(FRAME_PERIOD);
	}
      }
    }
      
    /*
      if ((slot%2000)<10)
      LOG_D(HW,"fun0: doing very hard work\n");
    */
    slot++;
    if (slot==20) {
      slot=0;
      frame++;
    }
#if defined(ENABLE_ITTI)
    itti_update_lte_time(frame, slot);
#endif
  }
  LOG_D(HW,"UE_thread: finished, ran %d times.\n",frame);

#ifdef HARD_RT
  rt_make_soft_real_time();
#endif

  // clean task
#ifdef RTAI
  rt_task_delete(task);
#endif
  LOG_D(HW,"Task deleted. returning\n");
  return 0;
}

#else  // This is for USRP or ETHERNET targets

#endif


static void get_options (int argc, char **argv) {
  int                           c;
  //  char                          line[1000];
  //  int                           l;
#ifdef USRP
  int clock_src;
#endif
  const Enb_properties_array_t *enb_properties;
  
  enum long_option_e {
    LONG_OPTION_START = 0x100, /* Start after regular single char options */
    
    LONG_OPTION_ULSCH_MAX_CONSECUTIVE_ERRORS,
    LONG_OPTION_CALIB_UE_RX,
    LONG_OPTION_CALIB_UE_RX_MED,
    LONG_OPTION_CALIB_UE_RX_BYP,
    
    LONG_OPTION_DEBUG_UE_PRACH,
    
    LONG_OPTION_NO_L2_CONNECT,
  };
  
  static const struct option long_options[] = {
    {"ulsch-max-errors",required_argument,  NULL, LONG_OPTION_ULSCH_MAX_CONSECUTIVE_ERRORS},
    {"calib-ue-rx",     required_argument,  NULL, LONG_OPTION_CALIB_UE_RX},
    {"calib-ue-rx-med", required_argument,  NULL, LONG_OPTION_CALIB_UE_RX_MED},
    {"calib-ue-rx-byp", required_argument,  NULL, LONG_OPTION_CALIB_UE_RX_BYP},
    {"debug-ue-prach",  no_argument,        NULL, LONG_OPTION_DEBUG_UE_PRACH},
    {"no-L2-connect",   no_argument,        NULL, LONG_OPTION_NO_L2_CONNECT},
    {NULL, 0, NULL, 0}};
  
  while ((c = getopt_long (argc, argv, "C:dK:qO:SUVRMr:s:",long_options,NULL)) != -1) {
    switch (c) {
    case LONG_OPTION_ULSCH_MAX_CONSECUTIVE_ERRORS:
      ULSCH_max_consecutive_errors = atoi(optarg);
      printf("Set ULSCH_max_consecutive_errors = %d\n",ULSCH_max_consecutive_errors);
      break;
      
    case LONG_OPTION_CALIB_UE_RX:
      mode = rx_calib_ue;
      rx_input_level_dBm = atoi(optarg);
      printf("Running with UE calibration on (LNA max), input level %d dBm\n",rx_input_level_dBm);
      break;
      
    case LONG_OPTION_CALIB_UE_RX_MED:
      mode = rx_calib_ue_med;
      rx_input_level_dBm = atoi(optarg);
      printf("Running with UE calibration on (LNA med), input level %d dBm\n",rx_input_level_dBm);
      break;
      
    case LONG_OPTION_CALIB_UE_RX_BYP:
      mode = rx_calib_ue_byp;
      rx_input_level_dBm = atoi(optarg);
      printf("Running with UE calibration on (LNA byp), input level %d dBm\n",rx_input_level_dBm);
      break;
      
    case LONG_OPTION_DEBUG_UE_PRACH:
      mode = debug_prach;
      break;
      
    case LONG_OPTION_NO_L2_CONNECT:
      mode = no_L2_connect;
      break;
    case 'M':
      multi_thread=0;
      break;
    case 'C':
      downlink_frequency[0] = atof(optarg); // Use float to avoid issue with frequency over 2^31.
      downlink_frequency[1] = downlink_frequency[0];
      downlink_frequency[2] = downlink_frequency[0];
      downlink_frequency[3] = downlink_frequency[0];
      carrier_freq[0] = downlink_frequency[0];
      carrier_freq[1] = downlink_frequency[1];
      carrier_freq[2] = downlink_frequency[2];
      carrier_freq[3] = downlink_frequency[3];
      printf("Downlink frequency set to %u\n", downlink_frequency[0]);
      break;
      
    case 'd':
#ifdef XFORMS
      do_forms=1;
#endif
      break;
      
    case 'K':
#if defined(ENABLE_ITTI)
      itti_dump_file = strdup(optarg);
#else
      printf("-K option is disabled when ENABLE_ITTI is not defined\n");
#endif
      break;
      
    case 'O':
      conf_config_file_name = optarg;
      break;
      
    case 'U':
      UE_flag = 1;
      break;
      
    case 'V':
      ouput_vcd = 1;
      break;
    case  'q': 
      opp_enabled = 1;
      break;
    case  'R' :
      online_log_messages =1;
      break;
    case 'r':
      switch(atoi(optarg)) {
      case 6:
	N_RB_DL=6;
	break;
      case 25:
	N_RB_DL=25;
	break;
      case 50:
	N_RB_DL=50;
	break;
      case 100:
	N_RB_DL=100;
	break;
      default:
	printf("Unknown N_RB_DL %d, switching to 25\n",atoi(optarg));
	break;
      }
    case 's':
#ifdef USRP

      clock_src = atoi(optarg);
      if (clock_src == 0) {
	char ref[128] = "internal";
	//strncpy(uhd_ref, ref, strlen(ref)+1);
      }
      else if (clock_src == 1) {
	char ref[128] = "external";
	//strncpy(uhd_ref, ref, strlen(ref)+1);
      }
#else
      printf("Note: -s not defined for ExpressMIMO2\n");
#endif
      break;
    default:
      break;
    }
  }
  
  if ((UE_flag == 0) && (conf_config_file_name != NULL)) {
    int i;
    
    NB_eNB_INST = 1;
    
    /* Read eNB configuration file */
    enb_properties = enb_config_init(conf_config_file_name);
    
    AssertFatal (NB_eNB_INST <= enb_properties->number,
		 "Number of eNB is greater than eNB defined in configuration file %s (%d/%d)!",
		 conf_config_file_name, NB_eNB_INST, enb_properties->number);
    
    /* Update some simulation parameters */
    frame_parms->frame_type =       enb_properties->properties[0]->frame_type;
    frame_parms->tdd_config =       enb_properties->properties[0]->tdd_config;
    frame_parms->tdd_config_S =     enb_properties->properties[0]->tdd_config_s;
    for (i = 0 ; i < (sizeof(downlink_frequency) / sizeof (downlink_frequency[0])); i++) {
      downlink_frequency[i] =       enb_properties->properties[0]->downlink_frequency;
      printf("Downlink frequency set to %u\n", downlink_frequency[i]);
      uplink_frequency_offset[i] =  enb_properties->properties[0]->uplink_frequency_offset;
    }
  }
}

int main(int argc, char **argv) {
#ifdef RTAI
  // RT_TASK *task;
#else
  int *eNB_thread_status_p;
  //  int *eNB_thread_status_rx[10],*eNB_thread_status_tx[10];
#endif
  int i,j,aa;
#if defined (XFORMS) || defined (EMOS) || (! defined (RTAI))
  void *status;
#endif
  

  uint16_t Nid_cell = 0;
  uint8_t  cooperation_flag=0, transmission_mode=1, abstraction_flag=0;
#ifndef OPENAIR2
  uint8_t beta_ACK=0,beta_RI=0,beta_CQI=2;
#endif

#ifdef ENABLE_TCXO
  unsigned int tcxo = 114;
#endif

  //  int amp;
  // uint8_t prach_fmt;
  // int N_ZC;

  //  int ret, ant;
  int ant_offset=0;

#if defined (EMOS) || (! defined (RTAI))
  int error_code;
#endif
  openair0_config_t openair0_cfg;

  set_latency_target();

  mode = normal_txrx;

  frame_parms = (LTE_DL_FRAME_PARMS*) malloc(sizeof(LTE_DL_FRAME_PARMS));
  /* Set some default values that may be overwritten while reading options */
  frame_parms->frame_type         = TDD; /* TDD */
  frame_parms->tdd_config         = 3;
  frame_parms->tdd_config_S       = 0;

  get_options (argc, argv); //Command-line options

  //randominit (0);
  set_taus_seed (0);

  // initialize the log (see log.h for details)
  logInit();

  set_glog(LOG_WARNING, LOG_MED);
  if (UE_flag==1)
    {
      printf("configuring for UE\n");

      set_comp_log(HW,      LOG_DEBUG,  LOG_HIGH, 1);
#ifdef OPENAIR2
      set_comp_log(PHY,     LOG_INFO,   LOG_HIGH, 1);
#else
      set_comp_log(PHY,     LOG_INFO,   LOG_HIGH, 1);
#endif
      set_comp_log(MAC,     LOG_INFO,   LOG_HIGH, 1);
      set_comp_log(RLC,     LOG_INFO,   LOG_HIGH, 1);
      set_comp_log(PDCP,    LOG_INFO,   LOG_HIGH, 1);
      set_comp_log(OTG,     LOG_INFO,   LOG_HIGH, 1);
      set_comp_log(RRC,     LOG_INFO,   LOG_HIGH, 1);
#if defined(ENABLE_ITTI)
      set_comp_log(EMU,     LOG_INFO,   LOG_MED, 1);
# if defined(ENABLE_USE_MME)
      set_comp_log(NAS,     LOG_INFO,   LOG_HIGH, 1);
# endif
#endif
    }
  else
    {
      printf("configuring for eNB\n");

      set_comp_log(HW,      LOG_DEBUG,  LOG_HIGH, 1);
#ifdef OPENAIR2
      set_comp_log(PHY,     LOG_INFO,   LOG_HIGH, 1);
#else
      set_comp_log(PHY,     LOG_INFO,   LOG_HIGH, 1);
#endif
      set_comp_log(MAC,     LOG_INFO,   LOG_HIGH, 1);
      set_comp_log(RLC,     LOG_TRACE,   LOG_HIGH, 1);
      set_comp_log(PDCP,    LOG_DEBUG,   LOG_HIGH, 1);
      set_comp_log(OTG,     LOG_INFO,   LOG_HIGH, 1);
      set_comp_log(RRC,     LOG_DEBUG,   LOG_HIGH, 1);
#if defined(ENABLE_ITTI)
      set_comp_log(EMU,     LOG_INFO,   LOG_MED, 1);
# if defined(ENABLE_USE_MME)
      set_comp_log(UDP_,    LOG_DEBUG,   LOG_HIGH, 1);
      set_comp_log(GTPU,    LOG_DEBUG,   LOG_HIGH, 1);
      set_comp_log(S1AP,    LOG_DEBUG,   LOG_HIGH, 1);
      set_comp_log(SCTP,    LOG_INFO,   LOG_HIGH, 1);
# endif
#if defined(ENABLE_SECURITY)
      set_comp_log(OSA,    LOG_DEBUG,   LOG_HIGH, 1);
#endif
#endif
      set_comp_log(ENB_APP, LOG_INFO, LOG_HIGH, 1);
      if (online_log_messages == 1) { 
	set_component_filelog(RRC);
	set_component_filelog(PDCP);
      }
    }

  if (ouput_vcd) {
    if (UE_flag==1)
      vcd_signal_dumper_init("/tmp/openair_dump_UE.vcd");
    else
      vcd_signal_dumper_init("/tmp/openair_dump_eNB.vcd");
  }

#if defined(ENABLE_ITTI)
  if (UE_flag == 1) {
    log_set_instance_type (LOG_INSTANCE_UE);
  }
  else {
    log_set_instance_type (LOG_INSTANCE_ENB);
  }

  itti_init(TASK_MAX, THREAD_MAX, MESSAGES_ID_MAX, tasks_info, messages_info, messages_definition_xml, itti_dump_file);
#endif

#ifdef NAS_NETLINK
  netlink_init();
#endif

#if !defined(ENABLE_ITTI)
  // to make a graceful exit when ctrl-c is pressed
  signal(SIGSEGV, signal_handler);
  signal(SIGINT, signal_handler);
#endif

#ifndef RTAI
  check_clock();
#endif

  // init the parameters
  frame_parms->N_RB_DL            = N_RB_DL;
  frame_parms->N_RB_UL            = N_RB_DL;
  frame_parms->Ncp                = NORMAL;
  frame_parms->Ncp_UL             = NORMAL;
  frame_parms->Nid_cell           = Nid_cell;
  frame_parms->nushift            = 0;
  if (UE_flag==0)
    {
      switch (transmission_mode) {
      case 1: 
	frame_parms->nb_antennas_tx     = 1;
	frame_parms->nb_antennas_rx     = 1;
	break;
      case 2:
      case 5:
      case 6:
	frame_parms->nb_antennas_tx     = 2;
	frame_parms->nb_antennas_rx     = 2;
	break;
      default:
	printf("Unsupported transmission mode %d\n",transmission_mode);
	exit(-1);
      }
    }
  else
    { //UE_flag==1
      frame_parms->nb_antennas_tx     = 1;
      frame_parms->nb_antennas_rx     = 1;
    }
  frame_parms->nb_antennas_tx_eNB = (transmission_mode == 1) ? 1 : 2; //initial value overwritten by initial sync later
  frame_parms->mode1_flag         = (transmission_mode == 1) ? 1 : 0;
  frame_parms->phich_config_common.phich_resource = oneSixth;
  frame_parms->phich_config_common.phich_duration = normal;
  // UL RS Config
  frame_parms->pusch_config_common.ul_ReferenceSignalsPUSCH.cyclicShift = 0;//n_DMRS1 set to 0
  frame_parms->pusch_config_common.ul_ReferenceSignalsPUSCH.groupHoppingEnabled = 0;
  frame_parms->pusch_config_common.ul_ReferenceSignalsPUSCH.sequenceHoppingEnabled = 0;
  frame_parms->pusch_config_common.ul_ReferenceSignalsPUSCH.groupAssignmentPUSCH = 0;
  init_ul_hopping(frame_parms);

  init_frame_parms(frame_parms,1);

  phy_init_top(frame_parms);
  phy_init_lte_top(frame_parms);

  //init prach for openair1 test
  frame_parms->prach_config_common.rootSequenceIndex=22; 
  frame_parms->prach_config_common.prach_ConfigInfo.zeroCorrelationZoneConfig=1;
  frame_parms->prach_config_common.prach_ConfigInfo.prach_ConfigIndex=0; 
  frame_parms->prach_config_common.prach_ConfigInfo.highSpeedFlag=0;
  frame_parms->prach_config_common.prach_ConfigInfo.prach_FreqOffset=0;
  // prach_fmt = get_prach_fmt(frame_parms->prach_config_common.prach_ConfigInfo.prach_ConfigIndex, frame_parms->frame_type);
  // N_ZC = (prach_fmt <4)?839:139;

  if (UE_flag==1) {
    PHY_vars_UE_g = malloc(sizeof(PHY_VARS_UE*));
    PHY_vars_UE_g[0] = init_lte_UE(frame_parms, UE_id,abstraction_flag,transmission_mode);
    
#ifndef OPENAIR2
    for (i=0;i<NUMBER_OF_eNB_MAX;i++) {
      PHY_vars_UE_g[0]->pusch_config_dedicated[i].betaOffset_ACK_Index = beta_ACK;
      PHY_vars_UE_g[0]->pusch_config_dedicated[i].betaOffset_RI_Index  = beta_RI;
      PHY_vars_UE_g[0]->pusch_config_dedicated[i].betaOffset_CQI_Index = beta_CQI;
      
      PHY_vars_UE_g[0]->scheduling_request_config[i].sr_PUCCH_ResourceIndex = UE_id;
      PHY_vars_UE_g[0]->scheduling_request_config[i].sr_ConfigIndex = 7+(UE_id%3);
      PHY_vars_UE_g[0]->scheduling_request_config[i].dsr_TransMax = sr_n4;
    }
#endif
    
    compute_prach_seq(&PHY_vars_UE_g[0]->lte_frame_parms.prach_config_common,
		      PHY_vars_UE_g[0]->lte_frame_parms.frame_type,
		      PHY_vars_UE_g[0]->X_u);
    
    PHY_vars_UE_g[0]->lte_ue_pdcch_vars[0]->crnti = 0x1234;
#ifndef OPENAIR2
    PHY_vars_UE_g[0]->lte_ue_pdcch_vars[0]->crnti = 0x1235;
#endif
    NB_UE_INST=1;
    NB_INST=1;
    
    openair_daq_vars.manual_timing_advance = 0;
    //openair_daq_vars.timing_advance = TIMING_ADVANCE_HW;
    openair_daq_vars.rx_gain_mode = DAQ_AGC_ON;
    openair_daq_vars.auto_freq_correction = 0;
    openair_daq_vars.use_ia_receiver = 0;
    
    for (i=0;i<4;i++) {
      PHY_vars_UE_g[0]->rx_gain_max[i] = rxg_max[i];
      //      PHY_vars_UE_g[0]->rx_gain_med[i] = rxg_med[i];
      //      PHY_vars_UE_g[0]->rx_gain_byp[i] = rxg_byp[i];
    }
    
    if ((mode == normal_txrx) || (mode == rx_calib_ue) || (mode == no_L2_connect) || (mode == debug_prach)) {
      for (i=0;i<4;i++)
	openair0_cfg.rxg_mode[i] =  max_gain;
      PHY_vars_UE_g[0]->rx_total_gain_dB =  PHY_vars_UE_g[0]->rx_gain_max[0] + (int)rx_gain - 30; //-30 because it was calibrated with a 30dB gain
    }
    else if ((mode == rx_calib_ue_med)) {
      for (i=0;i<4;i++)
	openair0_cfg.rxg_mode[i] =  med_gain;
      PHY_vars_UE_g[0]->rx_total_gain_dB =  PHY_vars_UE_g[0]->rx_gain_med[0]  + (int)rx_gain - 30; //-30 because it was calibrated with a 30dB gain;
    }
    else if ((mode == rx_calib_ue_byp)) {
      for (i=0;i<4;i++)
	openair0_cfg.rxg_mode[i] =  byp_gain;
      PHY_vars_UE_g[0]->rx_total_gain_dB =  PHY_vars_UE_g[0]->rx_gain_byp[0]  + (int)rx_gain - 30; //-30 because it was calibrated with a 30dB gain;
    }
    
    PHY_vars_UE_g[0]->tx_power_max_dBm = tx_max_power;
    
    //  printf("tx_max_power = %d -> amp %d\n",tx_max_power,get_tx_amp(tx_max_power,tx_max_power));
  }
  else
    { //this is eNB
      PHY_vars_eNB_g = malloc(sizeof(PHY_VARS_eNB*));
      PHY_vars_eNB_g[0] = init_lte_eNB(frame_parms,eNB_id,Nid_cell,cooperation_flag,transmission_mode,abstraction_flag);
    
#ifndef OPENAIR2
      for (i=0;i<NUMBER_OF_UE_MAX;i++) {
	PHY_vars_eNB_g[0]->pusch_config_dedicated[i].betaOffset_ACK_Index = beta_ACK;
	PHY_vars_eNB_g[0]->pusch_config_dedicated[i].betaOffset_RI_Index  = beta_RI;
	PHY_vars_eNB_g[0]->pusch_config_dedicated[i].betaOffset_CQI_Index = beta_CQI;

	PHY_vars_eNB_g[0]->scheduling_request_config[i].sr_PUCCH_ResourceIndex = i;
	PHY_vars_eNB_g[0]->scheduling_request_config[i].sr_ConfigIndex = 7+(i%3);
	PHY_vars_eNB_g[0]->scheduling_request_config[i].dsr_TransMax = sr_n4;
      }
#endif

      compute_prach_seq(&PHY_vars_eNB_g[0]->lte_frame_parms.prach_config_common,
			PHY_vars_eNB_g[0]->lte_frame_parms.frame_type,
			PHY_vars_eNB_g[0]->X_u);

      NB_eNB_INST=1;
      NB_INST=1;

      openair_daq_vars.ue_dl_rb_alloc=0x1fff;
      openair_daq_vars.target_ue_dl_mcs=16;
      openair_daq_vars.ue_ul_nb_rb=6;
      openair_daq_vars.target_ue_ul_mcs=6;

      // set eNB to max gain
      PHY_vars_eNB_g[0]->rx_total_gain_eNB_dB =  rxg_max[0] + (int)rx_gain - 30; //was measured at rxgain=30;
      for (i=0;i<4;i++)
	openair0_cfg.rxg_mode[i] =  max_gain;
    }



  dump_frame_parms(frame_parms);

  if(frame_parms->N_RB_DL == 100) {
    sample_rate = 30.72e6;
#ifdef USRP
    samples_per_packets = 2048;
    samples_per_frame = 307200;
    // from usrp_time_offset
    tx_forward_nsamps = 175;
    sf_bounds = sf_bounds_20;
    max_cnt = 150;
    tx_delay = 8;
#endif
  }
  else if(frame_parms->N_RB_DL == 50){
    sample_rate = 15.36e6;
#ifdef USRP
    samples_per_packets = 2048;
    samples_per_frame = 153600;
    tx_forward_nsamps = 95;
    sf_bounds = sf_bounds_10;
    max_cnt = 75;
    tx_delay = 4;
#endif
  }
  else if (frame_parms->N_RB_DL == 25) {
    sample_rate = 7.68e6;
#ifdef USRP
    samples_per_packets = 1024;
    samples_per_frame = 76800;
    tx_forward_nsamps = 70;
    sf_bounds = sf_bounds_5;
    max_cnt = 75;
    tx_delay = 4;
#endif
  }
  
  mac_xface = malloc(sizeof(MAC_xface));


  openair0_cfg.sample_rate = sample_rate;
  openair0_cfg.tx_num_channels = frame_parms->nb_antennas_tx;
  openair0_cfg.rx_num_channels = frame_parms->nb_antennas_rx;
  
  for (i=0;i<4;i++) {
    if (UE_flag==1) {
      openair0_cfg.tx_freq[i] = downlink_frequency[i]+uplink_frequency_offset[i];
      openair0_cfg.rx_freq[i] = downlink_frequency[i];
    }
    else {
      openair0_cfg.rx_freq[i] = downlink_frequency[i]+uplink_frequency_offset[i];
      openair0_cfg.tx_freq[i] = downlink_frequency[i];
    }
  }
  openair0_cfg.tx_bw = bw;
  openair0_cfg.rx_bw = bw;
  for (i=0;i<4;i++) {
    openair0_cfg.tx_gain[i] = tx_gain;
    openair0_cfg.rx_gain[i] = rx_gain;
  }
  
  if (openair0_device_init(&openair0, &openair0_cfg) <0) {
    printf("Exiting, cannot initialize device\n");
    exit(-1);
  }
#ifdef OPENAIR2
  int eMBMS_active=0;

  l2_init(frame_parms,eMBMS_active,
	  0,// cba_group_active
	  0); // HO flag
  if (UE_flag == 1)
    mac_xface->dl_phy_sync_success (0, 0, 0, 1);
  else
    mac_xface->mrbch_phy_sync_failure (0, 0, 0);
#endif

  mac_xface->macphy_exit = &exit_fun;

#if defined(ENABLE_ITTI)
  if (create_tasks(UE_flag ? 0 : 1, UE_flag ? 1 : 0) < 0) {
    exit(-1); // need a softer mode
  }
  printf("ITTI tasks created\n");
#endif

#ifdef OPENAIR2
  //if (otg_enabled) {
  init_all_otg(0);
  g_otg->seed = 0;
  init_seeds(g_otg->seed);
  g_otg->num_nodes = 2;
  for (i=0; i<g_otg->num_nodes; i++){
    for (j=0; j<g_otg->num_nodes; j++){
      g_otg->application_idx[i][j] = 1;
      //g_otg->packet_gen_type=SUBSTRACT_STRING;
      g_otg->aggregation_level[i][j][0]=1;
      g_otg->application_type[i][j][0] = BCBR; //MCBR, BCBR
    }
  }
  init_predef_traffic(UE_flag ? 1 : 0, UE_flag ? 0 : 1);
  //  }
#endif

  //  number_of_cards = openair0_num_detected_cards;

  openair_daq_vars.timing_advance = 0;

  // connect the TX/RX buffers
  if (UE_flag==1) {
    setup_ue_buffers(PHY_vars_UE_g[0],frame_parms,ant_offset);
    printf("Setting UE buffer to all-RX\n");
    // Set LSBs for antenna switch (ExpressMIMO)
    for (i=0; i<frame_parms->samples_per_tti*10; i++)
      for (aa=0; aa<frame_parms->nb_antennas_tx; aa++)
	PHY_vars_UE_g[0]->lte_ue_common_vars.txdata[aa][i] = 0x00010001;
    
    //p_exmimo_config->framing.tdd_config = TXRXSWITCH_TESTRX;
  }
  else {
    setup_eNB_buffers(PHY_vars_eNB_g[0],frame_parms,ant_offset);
    printf("Setting eNB buffer to all-RX\n");
    // Set LSBs for antenna switch (ExpressMIMO)
    for (i=0; i<frame_parms->samples_per_tti*10; i++)
      for (aa=0; aa<frame_parms->nb_antennas_tx; aa++)
	PHY_vars_eNB_g[0]->lte_eNB_common_vars.txdata[0][aa][i] = 0x00010001;
  }

#ifndef USRP
  openair0_dump_config(card);
#endif

  /*  
      for (ant=0;ant<4;ant++)
      p_exmimo_config->rf.do_autocal[ant] = 0;
  */

#ifdef EMOS
  error_code = rtf_create(CHANSOUNDER_FIFO_MINOR,CHANSOUNDER_FIFO_SIZE);
  if (error_code==0)
    printf("[OPENAIR][SCHED][INIT] Created EMOS FIFO %d\n",CHANSOUNDER_FIFO_MINOR);
  else if (error_code==ENODEV)
    printf("[OPENAIR][SCHED][INIT] Problem: EMOS FIFO %d is greater than or equal to RTF_NO\n",CHANSOUNDER_FIFO_MINOR);
  else if (error_code==ENOMEM)
    printf("[OPENAIR][SCHED][INIT] Problem: cannot allocate memory for EMOS FIFO %d\n",CHANSOUNDER_FIFO_MINOR);
  else 
    printf("[OPENAIR][SCHED][INIT] Problem creating EMOS FIFO %d, error_code %d\n",CHANSOUNDER_FIFO_MINOR,error_code);
#endif

  mlockall(MCL_CURRENT | MCL_FUTURE);

#ifdef RTAI
  // make main thread LXRT soft realtime
  /* task = */ rt_task_init_schmod(nam2num("MYTASK"), 9, 0, 0, SCHED_FIFO, 0xF);

  // start realtime timer and scheduler
  //rt_set_oneshot_mode();
  rt_set_periodic_mode();
  start_rt_timer(0);

  //now = rt_get_time() + 10*PERIOD;
  //rt_task_make_periodic(task, now, PERIOD);

  printf("Init mutex\n");
  //mutex = rt_get_adr(nam2num("MUTEX"));
  mutex = rt_sem_init(nam2num("MUTEX"), 1);
  if (mutex==0)
    {
      printf("Error init mutex\n");
      exit(-1);
    }
  else
    printf("mutex=%p\n",mutex);
#ifdef USRP
  sync_sem = rt_typed_sem_init(nam2num("syncsem"), 0, BIN_SEM|FIFO_Q);
  if(sync_sem == 0)
    printf("error init sync semphore\n");
#endif
#else
#ifdef USRP
  pthread_cond_init(&sync_cond,NULL);
  pthread_mutex_init(&sync_mutex, NULL);
#endif
#endif



  // this starts the DMA transfers
#ifndef USRP
  if (UE_flag!=1)
    openair0_start_rt_acquisition(card);
#endif

#ifdef XFORMS
  if (do_forms==1) {
    fl_initialize (&argc, argv, NULL, 0, 0);
    form_stats = create_form_stats_form();
    if (UE_flag==1) {
      form_ue[UE_id] = create_lte_phy_scope_ue();
      sprintf (title, "LTE DL SCOPE UE");
      fl_show_form (form_ue[UE_id]->lte_phy_scope_ue, FL_PLACE_HOTSPOT, FL_FULLBORDER, title);
    } else {
      for(UE_id=0;UE_id<scope_enb_num_ue;UE_id++) {
	form_enb[UE_id] = create_lte_phy_scope_enb();
	sprintf (title, "UE%d LTE UL SCOPE eNB",UE_id+1);
	fl_show_form (form_enb[UE_id]->lte_phy_scope_enb, FL_PLACE_HOTSPOT, FL_FULLBORDER, title);
      }
    }
    fl_show_form (form_stats->stats_form, FL_PLACE_HOTSPOT, FL_FULLBORDER, "stats");
    if (UE_flag==0) {
      for (UE_id=0;UE_id<scope_enb_num_ue;UE_id++) {
	if (otg_enabled) {
	  fl_set_button(form_enb[UE_id]->button_0,1);
	  fl_set_object_label(form_enb[UE_id]->button_0,"DL Traffic ON");
	}
	else {
	  fl_set_button(form_enb[UE_id]->button_0,0);
	  fl_set_object_label(form_enb[UE_id]->button_0,"DL Traffic OFF");
	}
      }
    }
    else {
      if (openair_daq_vars.use_ia_receiver) {
	fl_set_button(form_ue[UE_id]->button_0,1);
	fl_set_object_label(form_ue[UE_id]->button_0, "IA Receiver ON");
      }
      else {
	fl_set_button(form_ue[UE_id]->button_0,0);
	fl_set_object_label(form_ue[UE_id]->button_0, "IA Receiver OFF");
      }
    }

    ret = pthread_create(&thread2, NULL, scope_thread, NULL);
    printf("Scope thread created, ret=%d\n",ret);
  }
#endif

#ifdef EMOS
  ret = pthread_create(&thread3, NULL, emos_thread, NULL);
  printf("EMOS thread created, ret=%d\n",ret);
#endif

  rt_sleep_ns(10*FRAME_PERIOD);

#ifndef RTAI
  pthread_attr_init (&attr_dlsch_threads);
  pthread_attr_setstacksize(&attr_dlsch_threads,OPENAIR_THREAD_STACK_SIZE);
  //attr_dlsch_threads.priority = 1;
  sched_param_dlsch.sched_priority = 90;//sched_get_priority_max(SCHED_FIFO); //OPENAIR_THREAD_PRIORITY;
  pthread_attr_setschedparam  (&attr_dlsch_threads, &sched_param_dlsch);
  pthread_attr_setschedpolicy (&attr_dlsch_threads, SCHED_FIFO);
#endif

  // start the main thread
  if (UE_flag == 1) {
#ifndef USRP
#ifdef RTAI
    thread1 = rt_thread_create(UE_thread, NULL, 100000000);
#else
    error_code = pthread_create(&thread1, &attr_dlsch_threads, UE_thread, NULL);
    if (error_code!= 0) {
      LOG_D(HW,"[lte-softmodem.c] Could not allocate UE_thread, error %d\n",error_code);
      return(error_code);
    }
    else {
      LOG_D(HW,"[lte-softmodem.c] Allocate UE_thread successful\n");
    }
#endif
#ifdef DLSCH_THREAD
    init_rx_pdsch_thread();
    rt_sleep_ns(FRAME_PERIOD/10);
    init_dlsch_threads();
#endif
    printf("UE threads created\n");
#else
    printf("UE functionality not yet supported on USRP");
#endif
  }
  else {

    if (multi_thread>0) {
      init_eNB_proc();
      LOG_D(HW,"[lte-softmodem.c] eNB threads created\n");
    }
#ifdef RTAI
    thread0 = rt_thread_create(eNB_thread, NULL, 100000000);
#else
    error_code = pthread_create(&thread0, &attr_dlsch_threads, eNB_thread, NULL);
    if (error_code!= 0) {
      LOG_D(HW,"[lte-softmodem.c] Could not allocate eNB_thread, error %d\n",error_code);
      return(error_code);
    }
    else {
      LOG_D(HW,"[lte-softmodem.c] Allocate eNB_thread successful\n");
    }
#endif
  }

#ifdef USRP
  openair0.trx_start_func(&openair0);

#ifdef RTAI
  rt_sem_signal(sync_sem);
#else
  //pthread_mutex_lock(&sync_mutex);
  pthread_cond_signal(&sync_cond);
  //pthread_mutex_unlock(&sync_mutex);
#endif
#endif
  // wait for end of program
  printf("TYPE <CTRL-C> TO TERMINATE\n");
  //getchar();

#if defined(ENABLE_ITTI)
  printf("Entering ITTI signals handler\n");
  itti_wait_tasks_end();
#else
  while (oai_exit==0)
    rt_sleep_ns(FRAME_PERIOD);
#endif

  // stop threads
#ifdef XFORMS
  printf("waiting for XFORMS thread\n");
  if (do_forms==1)
    {
      pthread_join(thread2,&status);
      fl_hide_form(form_stats->stats_form);
      fl_free_form(form_stats->stats_form);
      if (UE_flag==1) {
	fl_hide_form(form_ue[UE_id]->lte_phy_scope_ue);
	fl_free_form(form_ue[UE_id]->lte_phy_scope_ue);
      } else {
	for(UE_id=0;UE_id<scope_enb_num_ue;UE_id++) {
	  fl_hide_form(form_enb[UE_id]->lte_phy_scope_enb);
	  fl_free_form(form_enb[UE_id]->lte_phy_scope_enb);
	}
      }
    }
#endif

  printf("stopping MODEM threads\n");
  // cleanup
  if (UE_flag == 1) {
#ifndef USRP
#ifdef RTAI
    rt_thread_join(thread1); 
#else
    pthread_join(thread1,&status); 
#endif
#ifdef DLSCH_THREAD
    cleanup_dlsch_threads();
    cleanup_rx_pdsch_thread();
#endif
#endif
  }
  else {
#ifdef RTAI
    rt_thread_join(thread0); 
#else
#ifdef DEBUG_THREADS
    printf("Joining eNB_thread ...");
#endif
    pthread_join(thread0,(void**)&eNB_thread_status_p); 
#ifdef DEBUG_THREADS
    printf("status %d\n",*eNB_thread_status_p);
#endif
#endif

    if (multi_thread>0) {
      printf("Killing eNB processing threads\n");
      kill_eNB_proc();

    }
  }

#ifdef OPENAIR2
  //cleanup_pdcp_thread();
#endif

#ifdef USRP
#ifdef RTAI
  rt_sem_delete(sync_sem);
  stop_rt_timer();
#else
  pthread_cond_destroy(&sync_cond);
  pthread_mutex_destroy(&sync_mutex);
#endif
#endif

#ifndef USRP
  printf("stopping card\n");
  openair0_stop(card);
  printf("closing openair0_lib\n");
  openair0_close();
#endif

#ifdef EMOS
  printf("waiting for EMOS thread\n");
  pthread_cancel(thread3);
  pthread_join(thread3,&status);
#endif

#ifdef EMOS
  error_code = rtf_destroy(CHANSOUNDER_FIFO_MINOR);
  printf("[OPENAIR][SCHED][CLEANUP] EMOS FIFO closed, error_code %d\n", error_code);
#endif

  if (ouput_vcd)
    vcd_signal_dumper_close();

  logClean();

  return 0;
}

void setup_ue_buffers(PHY_VARS_UE *phy_vars_ue, LTE_DL_FRAME_PARMS *frame_parms, int carrier) {

  int i;
#ifndef USRP
  if (phy_vars_ue) {
    if ((frame_parms->nb_antennas_rx>1) && (carrier>0)) {
      printf("RX antennas > 1 and carrier > 0 not possible\n");
      exit(-1);
    }

    if ((frame_parms->nb_antennas_tx>1) && (carrier>0)) {
      printf("TX antennas > 1 and carrier > 0 not possible\n");
      exit(-1);
    }
    
    // replace RX signal buffers with mmaped HW versions
    for (i=0;i<frame_parms->nb_antennas_rx;i++) {
      free(phy_vars_ue->lte_ue_common_vars.rxdata[i]);
      phy_vars_ue->lte_ue_common_vars.rxdata[i] = (int32_t*) openair0_exmimo_pci[card].adc_head[i+carrier];


      printf("rxdata[%d] @ %p\n",i,phy_vars_ue->lte_ue_common_vars.rxdata[i]);
    }
    for (i=0;i<frame_parms->nb_antennas_tx;i++) {
      free(phy_vars_ue->lte_ue_common_vars.txdata[i]);
      phy_vars_ue->lte_ue_common_vars.txdata[i] = (int32_t*) openair0_exmimo_pci[card].dac_head[i+carrier];

      printf("txdata[%d] @ %p\n",i,phy_vars_ue->lte_ue_common_vars.txdata[i]);
    }
  }
#else
  printf("USRP not supported for UE yet!");
#endif
}

void setup_eNB_buffers(PHY_VARS_eNB *phy_vars_eNB, LTE_DL_FRAME_PARMS *frame_parms, int carrier) {

  int i;
#ifdef USRP
  uint16_t N_TA_offset = 0;
#else
  int j;
#endif

  if (phy_vars_eNB) {
    if ((frame_parms->nb_antennas_rx>1) && (carrier>0)) {
      printf("RX antennas > 1 and carrier > 0 not possible\n");
      exit(-1);
    }

    if ((frame_parms->nb_antennas_tx>1) && (carrier>0)) {
      printf("TX antennas > 1 and carrier > 0 not possible\n");
      exit(-1);
    }

#ifdef USRP
    if (frame_parms->frame_type == TDD) {
      if (phy_vars_eNB->lte_frame_parms.N_RB_DL == 100)
	N_TA_offset = 624;
      else if (phy_vars_eNB->lte_frame_parms.N_RB_DL == 50)
	N_TA_offset = 624/2;
      else if (phy_vars_eNB->lte_frame_parms.N_RB_DL == 25)
	N_TA_offset = 624/4;
    }
#endif
    
    // replace RX signal buffers with mmaped HW versions
#ifndef USRP
    for (i=0;i<frame_parms->nb_antennas_rx;i++) {
      free(phy_vars_eNB->lte_eNB_common_vars.rxdata[0][i]);
      phy_vars_eNB->lte_eNB_common_vars.rxdata[0][i] = (int32_t*) openair0_exmimo_pci[card].adc_head[i+carrier];

      printf("rxdata[%d] @ %p\n",i,phy_vars_eNB->lte_eNB_common_vars.rxdata[0][i]);
      for (j=0;j<16;j++) {
	printf("rxbuffer %d: %x\n",j,phy_vars_eNB->lte_eNB_common_vars.rxdata[0][i][j]);
	phy_vars_eNB->lte_eNB_common_vars.rxdata[0][i][j] = 16-j;
      }
    }
    for (i=0;i<frame_parms->nb_antennas_tx;i++) {
      free(phy_vars_eNB->lte_eNB_common_vars.txdata[0][i]);
      phy_vars_eNB->lte_eNB_common_vars.txdata[0][i] = (int32_t*) openair0_exmimo_pci[card].dac_head[i+carrier];

      printf("txdata[%d] @ %p\n",i,phy_vars_eNB->lte_eNB_common_vars.txdata[0][i]);
      for (j=0;j<16;j++) {
	printf("txbuffer %d: %x\n",j,phy_vars_eNB->lte_eNB_common_vars.txdata[0][i][j]);
	phy_vars_eNB->lte_eNB_common_vars.txdata[0][i][j] = 16-j;
      }
    }
#else // USRP
    for (i=0;i<frame_parms->nb_antennas_rx;i++) {
        free(phy_vars_eNB->lte_eNB_common_vars.rxdata[0][i]);
        rxdata = (int32_t*)malloc16(samples_per_frame*sizeof(int32_t));
        phy_vars_eNB->lte_eNB_common_vars.rxdata[0][i] = rxdata-N_TA_offset; // N_TA offset for TDD
        memset(rxdata, 0, samples_per_frame*sizeof(int32_t));
        printf("rxdata[%d] @ %p\n", i, phy_vars_eNB->lte_eNB_common_vars.rxdata[0][i]);
    }
    for (i=0;i<frame_parms->nb_antennas_tx;i++) {
        free(phy_vars_eNB->lte_eNB_common_vars.txdata[0][i]);
        txdata = (int32_t*)malloc16(samples_per_frame*sizeof(int32_t));
        phy_vars_eNB->lte_eNB_common_vars.txdata[0][i] = txdata;
        memset(txdata, 0, samples_per_frame*sizeof(int32_t));
        printf("txdata[%d] @ %p\n", i, phy_vars_eNB->lte_eNB_common_vars.txdata[0][i]);
    }
#endif
  }
}