nghttpx: Use doubly linked list for dconns_ and streams_

src/shrpx_http2_downstream_connection.cc

@@ -44,8 +44,8 @@ namespace shrpx {
 
 Http2DownstreamConnection::Http2DownstreamConnection(
     DownstreamConnectionPool *dconn_pool, Http2Session *http2session)
-    : DownstreamConnection(dconn_pool), http2session_(http2session),
-      sd_(nullptr) {}
+    : DownstreamConnection(dconn_pool), dlnext(nullptr), dlprev(nullptr),
+      http2session_(http2session), sd_(nullptr) {}
 
 Http2DownstreamConnection::~Http2DownstreamConnection() {
   if (LOG_ENABLED(INFO)) {

src/shrpx_http2_downstream_connection.h

@@ -74,6 +74,8 @@ public:
   int submit_rst_stream(Downstream *downstream,
                         uint32_t error_code = NGHTTP2_INTERNAL_ERROR);
 
+  Http2DownstreamConnection *dlnext, *dlprev;
+
 private:
   Http2Session *http2session_;
   StreamData *sd_;

src/shrpx_http2_session.cc

@@ -44,7 +44,6 @@
 #include "http2.h"
 #include "util.h"
 #include "base64.h"
-#include "template.h"
 
 using namespace nghttp2;
 
@@ -208,13 +207,11 @@ int Http2Session::disconnect(bool hard) {
   // this object.  In order to achieve this, we first swap dconns_ and
   // streams_.  Upstream::on_downstream_reset() may add
   // Http2DownstreamConnection.
-  std::unordered_set<Http2DownstreamConnection *> dconns;
-  dconns.swap(dconns_);
-  std::unordered_set<StreamData *> streams;
-  streams.swap(streams_);
+  auto dconns = std::move(dconns_);
+  auto streams = std::move(streams_);
 
   std::set<ClientHandler *> handlers;
-  for (auto dc : dconns) {
+  for (auto dc = dconns.head; dc; dc = dc->dlnext) {
     if (!dc->get_client_handler()) {
       continue;
     }
@@ -226,8 +223,10 @@ int Http2Session::disconnect(bool hard) {
     }
   }
 
-  for (auto &s : streams) {
+  for (auto s = streams.head; s;) {
+    auto next = s->dlnext;
     delete s;
+    s = next;
   }
 
   return 0;
@@ -534,17 +533,17 @@ int Http2Session::downstream_connect_proxy() {
 }
 
 void Http2Session::add_downstream_connection(Http2DownstreamConnection *dconn) {
-  dconns_.insert(dconn);
+  dconns_.append(dconn);
 }
 
 void
 Http2Session::remove_downstream_connection(Http2DownstreamConnection *dconn) {
-  dconns_.erase(dconn);
+  dconns_.remove(dconn);
   dconn->detach_stream_data();
 }
 
 void Http2Session::remove_stream_data(StreamData *sd) {
-  streams_.erase(sd);
+  streams_.remove(sd);
   if (sd->dconn) {
     sd->dconn->detach_stream_data();
   }
@@ -556,6 +555,7 @@ int Http2Session::submit_request(Http2DownstreamConnection *dconn, int32_t pri,
                                  const nghttp2_data_provider *data_prd) {
   assert(state_ == CONNECTED);
   auto sd = make_unique<StreamData>();
+  sd->dlnext = sd->dlprev = nullptr;
   // TODO Specify nullptr to pri_spec for now
   auto stream_id =
       nghttp2_submit_request(session_, nullptr, nva, nvlen, data_prd, sd.get());
@@ -567,7 +567,7 @@ int Http2Session::submit_request(Http2DownstreamConnection *dconn, int32_t pri,
 
   dconn->attach_stream_data(sd.get());
   dconn->get_downstream()->set_downstream_stream_id(stream_id);
-  streams_.insert(sd.release());
+  streams_.append(sd.release());
 
   return 0;
 }
@@ -1498,7 +1498,7 @@ void Http2Session::connection_alive() {
 }
 
 void Http2Session::submit_pending_requests() {
-  for (auto dconn : dconns_) {
+  for (auto dconn = dconns_.head; dconn; dconn = dconn->dlnext) {
     auto downstream = dconn->get_downstream();
 
     if (!downstream || !downstream->request_submission_ready()) {

src/shrpx_http2_session.h

@@ -40,6 +40,7 @@
 
 #include "shrpx_connection.h"
 #include "buffer.h"
+#include "template.h"
 
 using namespace nghttp2;
 
@@ -50,6 +51,7 @@ class Worker;
 class ConnectBlocker;
 
 struct StreamData {
+  StreamData *dlnext, *dlprev;
   Http2DownstreamConnection *dconn;
 };
 
@@ -188,8 +190,8 @@ private:
   // connection check has started, this timer is started again and
   // traps PING ACK timeout.
   ev_timer connchk_timer_;
-  std::unordered_set<Http2DownstreamConnection *> dconns_;
-  std::unordered_set<StreamData *> streams_;
+  DList<Http2DownstreamConnection> dconns_;
+  DList<StreamData> streams_;
   std::function<int(Http2Session &)> read_, write_;
   std::function<int(Http2Session &)> on_read_, on_write_;
   // Used to parse the response from HTTP proxy

src/template.h

@@ -80,6 +80,61 @@ template <typename T, typename F> bool test_flags(T t, F flags) {
   return (t & flags) == flags;
 }
 
+// doubly linked list of element T*.  T must have field T *dlprev and
+// T *dlnext, which point to previous element and next element in the
+// list respectively.
+template <typename T> struct DList {
+  DList() : head(nullptr), tail(nullptr) {}
+
+  DList(const DList &) = delete;
+
+  DList &operator=(const DList &) = delete;
+
+  DList(DList &&other) : head(other.head), tail(other.tail) {
+    other.head = other.tail = nullptr;
+  }
+
+  DList &operator=(DList &&other) {
+    if (this == &other) {
+      return *this;
+    }
+    head = other.head;
+    tail = other.tail;
+    other.head = other.tail = nullptr;
+    return *this;
+  }
+
+  void append(T *t) {
+    if (tail) {
+      tail->dlnext = t;
+      t->dlprev = tail;
+      tail = t;
+      return;
+    }
+    head = tail = t;
+  }
+
+  void remove(T *t) {
+    auto p = t->dlprev;
+    auto n = t->dlnext;
+    if (p) {
+      p->dlnext = n;
+    }
+    if (head == t) {
+      head = n;
+    }
+    if (n) {
+      n->dlprev = p;
+    }
+    if (tail == t) {
+      tail = p;
+    }
+    t->dlprev = t->dlnext = nullptr;
+  }
+
+  T *head, *tail;
+};
+
 } // namespace nghttp2
 
 #endif // TEMPLATE_H