Bug deadlock SPGW-C context, itti health_ping (TODO)

src/common/utils/async_shell_cmd.cpp

@@ -78,17 +78,23 @@ void async_cmd_task (void* args_p)
         }
       }
       break;
+
     case TIME_OUT:
       if (itti_msg_timeout* to = dynamic_cast<itti_msg_timeout*>(msg)) {
         Logger::async_cmd().info( "TIME-OUT event timer id %d", to->timer_id);
       }
       break;
+
     case TERMINATE:
       if (itti_msg_terminate *terminate = dynamic_cast<itti_msg_terminate*>(msg)) {
         Logger::async_cmd().info( "Received terminate message");
         return;
       }
       break;
+
+    case HEALTH_PING:
+      break;
+
     default:
       Logger::sgwc_app().info( "no handler for msg type %d", msg->msg_type);
     }

src/gtpv2c/gtpv2c.cpp

@@ -632,6 +632,7 @@ void gtpv2c_stack::time_out_event(const uint32_t timer_id, const task_id_t& task
     if (it_proc != pending_procedures.end()) {
       if (it_proc->second.retry_count < GTPV2C_N3_REQUESTS) {
         it_proc->second.retry_count++;
+        it_proc->second.retry_timer_id = 0;
         start_msg_retry_timer(it_proc->second, GTPV2C_T3_RESPONSE_MS, task_id, it_proc->second.retry_msg->get_sequence_number());
         // send again message
         Logger::gtpv2_c().trace( "Retry %d Sending msg type %d, seq %d",

src/pgwc/pgw_app.cpp

@@ -120,13 +120,12 @@ void pgw_app::set_seid_2_pgw_context(const seid_t& seid, std::shared_ptr<pgw_con
   seid2pgw_context[seid] = pc;
 }
 //------------------------------------------------------------------------------
-bool pgw_app::seid_2_pgw_context(const seid_t& seid, std::shared_ptr<pgw_context>& pc, std::shared_lock<std::shared_mutex>& lock_found) const
+bool pgw_app::seid_2_pgw_context(const seid_t& seid, std::shared_ptr<pgw_context>& pc) const
 {
   std::shared_lock lock(m_seid2pgw_context);
   std::map<seid_t, std::shared_ptr<pgw_context>>::const_iterator it = seid2pgw_context.find(seid);
   if (it != seid2pgw_context.end()) {
     pc = it->second;
-    lock_found.swap(lock);
     return true;
   }
   return false;
@@ -259,6 +258,7 @@ void pgw_app_task (void*)
         Logger::pgwc_app().info( "Received terminate message");
         return;
       }
+    case HEALTH_PING:
       break;
     default:
       Logger::pgwc_app().info( "no handler for msg type %d", msg->msg_type);
@@ -564,10 +564,8 @@ void pgw_app::handle_itti_msg (itti_s5s8_downlink_data_notification_acknowledge&
 void pgw_app::handle_itti_msg (itti_sxab_session_establishment_response& seresp)
 {
   std::shared_ptr<pgw_context> pc = {};
-  std::shared_lock<std::shared_mutex> lpc;
-  if (seid_2_pgw_context(seresp.seid, pc, lpc)) {
+  if (seid_2_pgw_context(seresp.seid, pc)) {
     pc.get()->handle_itti_msg(seresp);
-    lpc.unlock();
   } else {
     Logger::pgwc_app().debug("Received SXAB SESSION ESTABLISHMENT RESPONSE seid" TEID_FMT "  pfcp_tx_id %" PRIX64", pgw_context not found, discarded!", seresp.seid, seresp.trxn_id);
   }
@@ -576,10 +574,8 @@ void pgw_app::handle_itti_msg (itti_sxab_session_establishment_response& seresp)
 void pgw_app::handle_itti_msg (itti_sxab_session_modification_response& smresp)
 {
   std::shared_ptr<pgw_context> pc = {};
-  std::shared_lock<std::shared_mutex> lpc;
-  if (seid_2_pgw_context(smresp.seid, pc, lpc)) {
+  if (seid_2_pgw_context(smresp.seid, pc)) {
     pc.get()->handle_itti_msg(smresp);
-    lpc.unlock();
   } else {
     Logger::pgwc_app().debug("Received SXAB SESSION MODIFICATION RESPONSE seid" TEID_FMT "  pfcp_tx_id %" PRIX64", pgw_context not found, discarded!", smresp.seid, smresp.trxn_id);
   }
@@ -588,10 +584,12 @@ void pgw_app::handle_itti_msg (itti_sxab_session_modification_response& smresp)
 void pgw_app::handle_itti_msg (itti_sxab_session_deletion_response& smresp)
 {
   std::shared_ptr<pgw_context> pc = {};
-  std::shared_lock<std::shared_mutex> lpc;
-  if (seid_2_pgw_context(smresp.seid, pc, lpc)) {
+  if (seid_2_pgw_context(smresp.seid, pc)) {
     pc.get()->handle_itti_msg(smresp);
-    lpc.unlock();
+
+    if (pc->apns.size() == 0) {
+      delete_pgw_context(pc);
+    }
   } else {
     Logger::pgwc_app().debug("Received SXAB SESSION DELETION RESPONSE seid" TEID_FMT "  pfcp_tx_id %" PRIX64", pgw_context not found, discarded!", smresp.seid, smresp.trxn_id);
   }
@@ -601,10 +599,8 @@ void pgw_app::handle_itti_msg (itti_sxab_session_deletion_response& smresp)
 void pgw_app::handle_itti_msg (std::shared_ptr<itti_sxab_session_report_request> snr)
 {
   std::shared_ptr<pgw_context> pc = {};
-  std::shared_lock<std::shared_mutex> lpc;
-  if (seid_2_pgw_context(snr->seid, pc, lpc)) {
+  if (seid_2_pgw_context(snr->seid, pc)) {
     pc.get()->handle_itti_msg(snr);
-    lpc.unlock();
   } else {
     Logger::pgwc_app().debug("Received SXAB SESSION REPORT REQUEST seid" TEID_FMT "  pfcp_tx_id %" PRIX64", pgw_context not found, discarded!", snr->seid, snr->trxn_id);
   }

src/pgwc/pgw_app.hpp

@@ -105,7 +105,7 @@ public:
   std::shared_ptr<pgw_context> s5s8cpgw_fteid_2_pgw_context(fteid_t& ls5s8_fteid);
 
   void set_seid_2_pgw_context(const seid_t& seid, std::shared_ptr<pgw_context>& pc);
-  bool seid_2_pgw_context(const seid_t& seid, std::shared_ptr<pgw_context>& pc, std::shared_lock<std::shared_mutex>& lock_found) const;
+  bool seid_2_pgw_context(const seid_t& seid, std::shared_ptr<pgw_context>& pc) const;
 
   void delete_pgw_context(std::shared_ptr<pgw_context> spc);
 

src/pgwc/pgw_context.hpp

@@ -31,7 +31,6 @@
 #include <map>
 #include <mutex>
 #include <memory>
-#include <shared_mutex>
 #include <utility>
 #include <vector>
 
@@ -241,15 +240,15 @@ public:
 class apn_context {
 
 public:
-  apn_context() : m_pdn_connections(), in_use(false), pdn_connections() {
+  apn_context() : m_context(), in_use(false), pdn_connections() {
     apn_ambr = {0};
   }
 
   apn_context(apn_context& b) = delete;
 
   void insert_pdn_connection(std::shared_ptr<pgw_pdn_connection>& sp);
-  bool find_pdn_connection(const teid_t xgw_s5s8c_teid, const bool is_local_teid, std::shared_ptr<pgw_pdn_connection>& pdn, std::shared_lock<std::shared_mutex>& lock_found);
-  bool find_pdn_connection(const pfcp::pdr_id_t& pdr_id, std::shared_ptr<pgw_pdn_connection>& pdn, ebi_t& ebi, std::shared_lock<std::shared_mutex>& lock_found);
+  bool find_pdn_connection(const teid_t xgw_s5s8c_teid, const bool is_local_teid, std::shared_ptr<pgw_pdn_connection>& pdn);
+  bool find_pdn_connection(const pfcp::pdr_id_t& pdr_id, std::shared_ptr<pgw_pdn_connection>& pdn, ebi_t& ebi);
   void delete_pdn_connection(std::shared_ptr<pgw_pdn_connection>& pdn_connection);
   int get_num_pdn_connections() const {return pdn_connections.size();};
   // deallocate_ressources is for releasing LTE resources prior to the deletion of objects
@@ -268,8 +267,7 @@ public:
   // key is local s5s8 teid
   //map<teid_t, shared_ptr<pgw_pdn_connection>> pdn_connections;
   std::vector<std::shared_ptr<pgw_pdn_connection>> pdn_connections; // was list
-  mutable std::shared_mutex                      m_pdn_connections;
-
+  mutable std::recursive_mutex                     m_context;
 };
 
 class pgw_context;
@@ -279,24 +277,23 @@ typedef std::pair<std::shared_ptr<apn_context>, std::shared_ptr<pgw_pdn_connecti
 class pgw_context : public std::enable_shared_from_this<pgw_context> {
 
 public:
-  pgw_context() : m_pending_procedures(), m_apns(),
-      imsi(), imsi_unauthenticated_indicator(false), apns(), pending_procedures(), msisdn() {}
+  pgw_context() : m_context(), imsi(), imsi_unauthenticated_indicator(false), apns(), pending_procedures(), msisdn() {}
 
   pgw_context(pgw_context& b) = delete;
 
   //void create_procedure(itti_s5s8_create_session_request& csreq);
   void insert_procedure(std::shared_ptr<pgw_procedure>& sproc);
-  bool find_procedure(const uint64_t& trxn_id, std::shared_ptr<pgw_procedure>& proc, std::shared_lock<std::shared_mutex>& lock_found);
+  bool find_procedure(const uint64_t& trxn_id, std::shared_ptr<pgw_procedure>& proc);
   void remove_procedure(pgw_procedure* proc);
 
 #define IS_FIND_PDN_WITH_LOCAL_TEID true
 #define IS_FIND_PDN_WITH_PEER_TEID  false
 
-  bool find_pdn_connection(const std::string& apn, const teid_t xgw_s5s8c_teid, const bool is_local_teid, pdn_duo_t& pdn_connection, std::shared_lock<std::shared_mutex>& lock_found);
-  bool find_pdn_connection(const teid_t xgw_s5s8c_teid, const bool is_local_teid, pdn_duo_t& pdn_connection, std::shared_lock<std::shared_mutex>& lock_found);
-  bool find_pdn_connection(const pfcp::pdr_id_t& pdr_id, std::shared_ptr<pgw_pdn_connection>& pdn, ebi_t& ebi, std::shared_lock<std::shared_mutex>& lock_found);
+  bool find_pdn_connection(const std::string& apn, const teid_t xgw_s5s8c_teid, const bool is_local_teid, pdn_duo_t& pdn_connection);
+  bool find_pdn_connection(const teid_t xgw_s5s8c_teid, const bool is_local_teid, pdn_duo_t& pdn_connection);
+  bool find_pdn_connection(const pfcp::pdr_id_t& pdr_id, std::shared_ptr<pgw_pdn_connection>& pdn, ebi_t& ebi);
   void insert_apn(std::shared_ptr<apn_context>& sa);
-  bool find_apn_context(const std::string& apn, std::shared_ptr<apn_context>& apn_context, std::shared_lock<std::shared_mutex>& lock_found);
+  bool find_apn_context(const std::string& apn, std::shared_ptr<apn_context>& apn_context);
   int get_num_apn_contexts() {return apns.size();};
 
   void delete_apn_context(std::shared_ptr<apn_context>& sa);
@@ -327,12 +324,13 @@ public:
   // NOT IMPLEMENTED OMC identity                           // Identifies the OMC that shall receive the trace record(s).
 
   std::vector<std::shared_ptr<apn_context>> apns; // was list
-  mutable std::shared_mutex               m_apns;
 
   //--------------------------------------------
   // internals
   std::vector<std::shared_ptr<pgw_procedure>> pending_procedures;
-  mutable std::shared_mutex                   m_pending_procedures;
+
+  // Big recursive lock
+  mutable std::recursive_mutex                m_context;
 };
 }
 

src/pgwc/pgw_s5s8.cpp

@@ -100,6 +100,9 @@ void pgw_s5s8_task (void *args_p)
       }
       break;
 
+    case HEALTH_PING:
+      break;
+
     default:
       Logger::pgwc_s5s8().info( "no handler for msg type %d", msg->msg_type);
     }
@@ -163,6 +166,8 @@ void pgw_s5s8::handle_receive_create_session_request(gtpv2c_msg& msg, const endp
     if (RETURNok != ret) {
       Logger::pgwc_s5s8().error( "Could not send ITTI message %s to task TASK_PGWC_APP", i->get_msg_name());
     }
+  } else {
+    Logger::pgwc_s5s8().error( "Error signalled by lower layer while receiving CREATE_SESSION_REQUEST");
   }
   // else ignore
 }

src/pgwc/pgwc_procedure.cpp

@@ -932,6 +932,14 @@ void delete_session_procedure::handle_itti_msg (itti_sxab_session_deletion_respo
   if (RETURNok != ret) {
     Logger::pgwc_app().error( "Could not send ITTI message %s to task TASK_PGWC_S5S8", s5_triggered_pending->gtp_ies.get_msg_name());
   }
+
+  // find APN context
+  pdn_duo_t pdn_duo = {};
+  if (pc->find_pdn_connection(ppc->pgw_fteid_s5_s8_cp.teid_gre_key, IS_FIND_PDN_WITH_LOCAL_TEID, pdn_duo)) {
+    pc->delete_pdn_connection(pdn_duo.first, pdn_duo.second);
+  } else {
+    Logger::pgwc_app().error("Could not delete PDN connection (APN context not found)");
+  }
 }
 //------------------------------------------------------------------------------
 int downlink_data_report_procedure::run(std::shared_ptr<pgwc::pgw_context> context,

src/pgwc/pgwc_procedure.hpp

@@ -149,8 +149,8 @@ public:
 //------------------------------------------------------------------------------
 class delete_session_procedure : public pgw_procedure {
 public:
-  explicit delete_session_procedure(std::shared_ptr<pgw_pdn_connection>& sppc) : pgw_procedure(), ppc(sppc),
-    sx_triggered(), s5_triggered_pending(), s5_trigger() {}
+  explicit delete_session_procedure(std::shared_ptr<pgw_context> spc, std::shared_ptr<pgw_pdn_connection>& sppc) : pgw_procedure(), ppc(sppc),
+    pc(spc), sx_triggered(), s5_triggered_pending(), s5_trigger() {}
 
   int run(std::shared_ptr<itti_s5s8_delete_session_request>& req,
           std::shared_ptr<itti_s5s8_delete_session_response>&resp,

src/pgwc/pgwc_sxab.cpp

@@ -168,6 +168,10 @@ void pgwc_sxab_task (void *args_p)
         return;
       }
       break;
+
+    case HEALTH_PING:
+      break;
+
     default:
       Logger::pgwc_sx().info( "no handler for msg type %d", msg->msg_type);
     }

src/sgwc/sgwc_app.cpp

@@ -137,6 +137,12 @@ void sgwc_app::set_s5s8sgw_teid_2_sgw_contexts(const teid_t& sgw_teid, shared_pt
 {
   s5s8lteid2sgw_contexts[sgw_teid] = std::make_pair(sebc, spc);
 }
+//------------------------------------------------------------------------------
+void sgwc_app::delete_s5s8sgw_teid_2_sgw_contexts(const teid_t& sgw_teid)
+{
+  s5s8lteid2sgw_contexts.erase(sgw_teid);
+}
+
 //------------------------------------------------------------------------------
 void sgwc_app::set_s11sgw_teid_2_sgw_eps_bearer_context(const teid_t& sgw_teid, shared_ptr<sgw_eps_bearer_context> sebc)
 {
@@ -255,6 +261,10 @@ void sgwc_app_task (void *args_p)
         return;
       }
       break;
+
+    case HEALTH_PING:
+      break;
+
     default:
       Logger::sgwc_app().info( "no handler for ITTI msg type %d", msg->msg_type);
     }

src/sgwc/sgwc_app.hpp

@@ -109,6 +109,7 @@ public:
   void delete_sgw_eps_bearer_context(std::shared_ptr<sgw_eps_bearer_context> sebc);
   std::shared_ptr<sgw_eps_bearer_context> imsi64_2_sgw_eps_bearer_context(const imsi64_t& imsi64) const;
   void set_s5s8sgw_teid_2_sgw_contexts(const teid_t& sgw_teid, std::shared_ptr<sgw_eps_bearer_context> sebc, std::shared_ptr<sgw_pdn_connection> spc);
+  void delete_s5s8sgw_teid_2_sgw_contexts(const teid_t& sgw_teid);
 
   void handle_itti_msg (itti_s11_create_session_request& m);
   void handle_itti_msg (itti_s11_delete_session_request& m);

src/sgwc/sgwc_eps_bearer_context.cpp

@@ -255,6 +255,7 @@ void sgw_eps_bearer_context::delete_pdn_connection(std::shared_ptr<sgw_pdn_conne
 {
   if (spc.get()) {
     Logger::sgwc_app().debug("sgw_eps_bearer_context::delete_pdn_connection() OK doing it");
+    sgwc_app_inst->delete_s5s8sgw_teid_2_sgw_contexts(spc->sgw_fteid_s5_s8_cp.teid_gre_key);
     erase_pdn_connection(spc);
     spc.get()->deallocate_ressources();
     spc.get()->delete_bearers();

src/sgwc/sgwc_s11.cpp

@@ -116,6 +116,9 @@ void sgw_s11_task (void *args_p)
       }
       break;
 
+    case HEALTH_PING:
+      break;
+
     default:
       Logger::sgwc_s11().info( "no handler for msg type %d", msg->msg_type);
     }

src/sgwc/sgwc_s5s8.cpp

@@ -99,6 +99,9 @@ void sgw_s5s8_task (void *args_p)
       }
       break;
 
+    case HEALTH_PING:
+      break;
+
     default:
       Logger::sgwc_s5s8().info( "no handler for msg type %d", msg->msg_type);
     }

src/spgwu/simpleswitch/spgwu_s1u.cpp

@@ -77,12 +77,17 @@ void spgwu_s1u_task (void *args_p)
         Logger::spgwu_s1u().info( "TIME-OUT event timer id %d", to->timer_id);
       }
       break;
+
     case TERMINATE:
       if (itti_msg_terminate *terminate = dynamic_cast<itti_msg_terminate*>(msg)) {
         Logger::spgwu_s1u().info( "Received terminate message");
         return;
       }
       break;
+
+    case HEALTH_PING:
+      break;
+
     default:
       Logger::spgwu_s1u().info( "no handler for msg type %d", msg->msg_type);
     }

src/spgwu/spgwu_app.cpp

@@ -85,6 +85,10 @@ void spgwu_app_task (void *args_p)
       spgwu_app_inst->handle_itti_msg(std::static_pointer_cast<itti_sxab_session_deletion_request>(shared_msg));
       break;
 
+    case SXAB_SESSION_REPORT_RESPONSE:
+      spgwu_app_inst->handle_itti_msg(std::static_pointer_cast<itti_sxab_session_report_response>(shared_msg));
+      break;
+
     case TIME_OUT:
       if (itti_msg_timeout* to = dynamic_cast<itti_msg_timeout*>(msg)) {
         switch (to->arg1_user) {
@@ -99,12 +103,17 @@ void spgwu_app_task (void *args_p)
         }
       }
       break;
+
     case TERMINATE:
       if (itti_msg_terminate *terminate = dynamic_cast<itti_msg_terminate*>(msg)) {
         Logger::spgwu_app().info( "Received terminate message");
         return;
       }
       break;
+
+    case HEALTH_PING:
+      break;
+
     default:
       Logger::spgwu_app().info( "no handler for ITTI msg type %d", msg->msg_type);
     }
@@ -218,4 +227,10 @@ void spgwu_app::handle_itti_msg (std::shared_ptr<itti_sxab_session_deletion_requ
   }
 }
 
+//------------------------------------------------------------------------------
+void spgwu_app::handle_itti_msg (std::shared_ptr<itti_sxab_session_report_response> m)
+{
+  Logger::spgwu_app().info("Received SXAB_SESSION_REPORT_RESPONSE seid " SEID_FMT " ", m->seid);
+}
+
 

src/spgwu/spgwu_app.hpp

@@ -79,7 +79,7 @@ public:
   void handle_itti_msg (std::shared_ptr<itti_sxab_session_deletion_request> m);
 //  void handle_itti_msg (itti_sxab_session_deletion_response& m);
 //  void handle_itti_msg (itti_sxab_session_report_request& m);
-//  void handle_itti_msg (itti_sxab_session_report_response& m);
+  void handle_itti_msg (std::shared_ptr<itti_sxab_session_report_response> m);
 };
 }
 #endif /* FILE_SPGWU_APP_HPP_SEEN */

src/spgwu/spgwu_sx.cpp

@@ -364,6 +364,29 @@ void spgwu_sx::handle_receive_session_deletion_request(pfcp_msg& msg, const endp
   }
   // else ignore
 }
+//------------------------------------------------------------------------------
+void spgwu_sx::handle_receive_session_report_response(pfcp_msg& msg, const endpoint& remote_endpoint)
+{
+  bool error = true;
+  uint64_t trxn_id = 0;
+  pfcp_session_report_response msg_ies_container = {};
+  msg.to_core_type(msg_ies_container);
+
+  handle_receive_message_cb(msg, remote_endpoint, TASK_SPGWU_SX, error, trxn_id);
+  if (!error) {
+    itti_sxab_session_report_response *itti_msg = new itti_sxab_session_report_response(TASK_SPGWU_SX, TASK_SPGWU_APP);
+    itti_msg->pfcp_ies = msg_ies_container;
+    itti_msg->r_endpoint = remote_endpoint;
+    itti_msg->trxn_id = trxn_id;
+    itti_msg->seid = msg.get_seid();
+    std::shared_ptr<itti_sxab_session_report_response> i = std::shared_ptr<itti_sxab_session_report_response>(itti_msg);
+    int ret = itti_inst->send_msg(i);
+    if (RETURNok != ret) {
+      Logger::spgwu_sx().error( "Could not send ITTI message %s to task TASK_SPGWU_APP", i->get_msg_name());
+    }
+  }
+  // else ignore ?
+}
 
 //------------------------------------------------------------------------------
 void spgwu_sx::handle_receive_pfcp_msg(pfcp_msg& msg, const endpoint& remote_endpoint)
@@ -387,6 +410,10 @@ void spgwu_sx::handle_receive_pfcp_msg(pfcp_msg& msg, const endpoint& remote_end
     handle_receive_session_deletion_request(msg, remote_endpoint);
     break;
 
+  case PFCP_SESSION_REPORT_RESPONSE:
+    handle_receive_session_report_response(msg, remote_endpoint);
+    break;
+
   case PFCP_HEARTBEAT_REQUEST:
     handle_receive_heartbeat_request(msg, remote_endpoint);
     break;
@@ -411,7 +438,6 @@ void spgwu_sx::handle_receive_pfcp_msg(pfcp_msg& msg, const endpoint& remote_end
   case PFCP_SESSION_MODIFICATION_RESPONSE:
   case PFCP_SESSION_DELETION_RESPONSE:
   case PFCP_SESSION_REPORT_REQUEST:
-  case PFCP_SESSION_REPORT_RESPONSE:
     Logger::spgwu_sx().info( "handle_receive_pfcp_msg msg %d length %d, not handled, discarded!", msg.get_message_type(), msg.get_message_length());
     break;
   default:

src/spgwu/spgwu_sx.hpp

@@ -105,8 +105,7 @@ public:
   void handle_receive_session_establishment_request(pfcp::pfcp_msg& msg, const endpoint& remote_endpoint);
   void handle_receive_session_modification_request(pfcp::pfcp_msg& msg, const endpoint& remote_endpoint);
   void handle_receive_session_deletion_request(pfcp::pfcp_msg& msg, const endpoint& remote_endpoint);
-
-
+  void handle_receive_session_report_response(pfcp::pfcp_msg& msg, const endpoint& remote_endpoint);
 
   void time_out_itti_event(const uint32_t timer_id);
 };