shrpx: Share SPDY session among multiple frontend connections per thread

In client mode, now SPDY connection to the backend server is
established per thread.  The frontend connections which belong to the
same thread share the SPDY connection.

src/Makefile.am

@@ -84,6 +84,7 @@ shrpx_SOURCES = ${HELPER_OBJECTS} ${HELPER_HFILES} \
 	shrpx_downstream_connection.cc shrpx_downstream_connection.h \
 	shrpx_http_downstream_connection.cc shrpx_http_downstream_connection.h \
 	shrpx_spdy_downstream_connection.cc shrpx_spdy_downstream_connection.h \
+	shrpx_spdy_session.cc shrpx_spdy_session.h \
 	shrpx_log.cc shrpx_log.h \
 	shrpx_http.cc shrpx_http.h \
 	shrpx_io_control.cc shrpx_io_control.h \

src/shrpx.cc

@@ -252,6 +252,8 @@ int event_loop()
 
   if(get_config()->num_worker > 1) {
     listener_handler->create_worker_thread(get_config()->num_worker);
+  } else if(get_config()->client_mode) {
+    listener_handler->create_spdy_session();
   }
 
   if(ENABLE_LOG) {

src/shrpx_client_handler.cc

@@ -118,11 +118,11 @@ ClientHandler::ClientHandler(bufferevent *bev, int fd, SSL *ssl,
                              const char *ipaddr)
   : bev_(bev),
     fd_(fd),
-    ssl_client_ctx_(0),
     ssl_(ssl),
     upstream_(0),
     ipaddr_(ipaddr),
-    should_close_after_write_(false)
+    should_close_after_write_(false),
+    spdy_(0)
 {
   bufferevent_enable(bev_, EV_READ | EV_WRITE);
   bufferevent_setwatermark(bev_, EV_READ, 0, SHRPX_READ_WARTER_MARK);
@@ -294,14 +294,14 @@ SSL* ClientHandler::get_ssl() const
   return ssl_;
 }
 
-void ClientHandler::set_ssl_client_ctx(SSL_CTX *ssl_ctx)
+void ClientHandler::set_spdy_session(SpdySession *spdy)
 {
-  ssl_client_ctx_ = ssl_ctx;
+  spdy_ = spdy;
 }
 
-SSL_CTX* ClientHandler::get_ssl_client_ctx() const
+SpdySession* ClientHandler::get_spdy_session() const
 {
-  return ssl_client_ctx_;
+  return spdy_;
 }
 
 } // namespace shrpx

src/shrpx_client_handler.h

@@ -36,6 +36,7 @@ namespace shrpx {
 
 class Upstream;
 class DownstreamConnection;
+class SpdySession;
 
 class ClientHandler {
 public:
@@ -60,19 +61,19 @@ public:
   DownstreamConnection* get_downstream_connection();
   size_t get_pending_write_length();
   SSL* get_ssl() const;
-  void set_ssl_client_ctx(SSL_CTX *ssl_ctx);
-  SSL_CTX* get_ssl_client_ctx() const;
+  void set_spdy_session(SpdySession *spdy);
+  SpdySession* get_spdy_session() const;
 private:
   bufferevent *bev_;
   int fd_;
-  // SSL_CTX for SSL object to connect backend SPDY server
-  SSL_CTX *ssl_client_ctx_;
   SSL *ssl_;
   Upstream *upstream_;
   std::string ipaddr_;
   bool should_close_after_write_;
-
   std::set<DownstreamConnection*> dconn_pool_;
+  // Shared SPDY session for each thread. NULL if not client mode. Not
+  // deleted by this object.
+  SpdySession *spdy_;
 };
 
 } // namespace shrpx

src/shrpx_downstream.cc

@@ -42,7 +42,6 @@ Downstream::Downstream(Upstream *upstream, int stream_id, int priority)
     dconn_(0),
     stream_id_(stream_id),
     priority_(priority),
-    ioctrl_(0),
     downstream_stream_id_(-1),
     request_state_(INITIAL),
     request_major_(1),
@@ -58,13 +57,9 @@ Downstream::Downstream(Upstream *upstream, int stream_id, int priority)
     chunked_response_(false),
     response_connection_close_(false),
     response_header_key_prev_(false),
-    response_htp_(new http_parser()),
     response_body_buf_(0),
     recv_window_size_(0)
-{
-  http_parser_init(response_htp_, HTTP_RESPONSE);
-  response_htp_->data = this;
-}
+{}
 
 Downstream::~Downstream()
 {
@@ -78,7 +73,6 @@ Downstream::~Downstream()
   if(dconn_) {
     delete dconn_;
   }
-  delete response_htp_;
   if(ENABLE_LOG) {
     LOG(INFO) << "Deleted";
   }
@@ -87,11 +81,6 @@ Downstream::~Downstream()
 void Downstream::set_downstream_connection(DownstreamConnection *dconn)
 {
   dconn_ = dconn;
-  if(dconn_) {
-    ioctrl_.set_bev(dconn_->get_bev());
-  } else {
-    ioctrl_.set_bev(0);
-  }
 }
 
 DownstreamConnection* Downstream::get_downstream_connection()
@@ -101,17 +90,25 @@ DownstreamConnection* Downstream::get_downstream_connection()
 
 void Downstream::pause_read(IOCtrlReason reason)
 {
-  ioctrl_.pause_read(reason);
+  if(dconn_) {
+    dconn_->pause_read(reason);
+  }
 }
 
 bool Downstream::resume_read(IOCtrlReason reason)
 {
-  return ioctrl_.resume_read(reason);
+  if(dconn_) {
+    return dconn_->resume_read(reason);
+  } else {
+    return false;
+  }
 }
 
 void Downstream::force_resume_read()
 {
-  ioctrl_.force_resume_read();
+  if(dconn_) {
+    dconn_->force_resume_read();
+  }
 }
 
 namespace {
@@ -285,9 +282,7 @@ bool Downstream::get_expect_100_continue() const
 bool Downstream::get_output_buffer_full()
 {
   if(dconn_) {
-    bufferevent *bev = dconn_->get_bev();
-    evbuffer *output = bufferevent_get_output(bev);
-    return evbuffer_get_length(output) >= DOWNSTREAM_OUTPUT_UPPER_THRES;
+    return dconn_->get_output_buffer_full();
   } else {
     return false;
   }
@@ -414,116 +409,9 @@ void Downstream::set_response_connection_close(bool f)
   response_connection_close_ = f;
 }
 
-namespace {
-int htp_hdrs_completecb(http_parser *htp)
-{
-  Downstream *downstream;
-  downstream = reinterpret_cast<Downstream*>(htp->data);
-  downstream->set_response_http_status(htp->status_code);
-  downstream->set_response_major(htp->http_major);
-  downstream->set_response_minor(htp->http_minor);
-  downstream->set_response_connection_close(!http_should_keep_alive(htp));
-  downstream->set_response_state(Downstream::HEADER_COMPLETE);
-  if(downstream->get_upstream()->on_downstream_header_complete(downstream)
-     != 0) {
-    return -1;
-  }
-  unsigned int status = downstream->get_response_http_status();
-  // Ignore the response body. HEAD response may contain
-  // Content-Length or Transfer-Encoding: chunked.  Some server send
-  // 304 status code with nonzero Content-Length, but without response
-  // body. See
-  // http://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-20#section-3.3
-  return downstream->get_request_method() == "HEAD" ||
-    (100 <= status && status <= 199) || status == 204 ||
-    status == 304 ? 1 : 0;
-}
-} // namespace
-
-namespace {
-int htp_hdr_keycb(http_parser *htp, const char *data, size_t len)
-{
-  Downstream *downstream;
-  downstream = reinterpret_cast<Downstream*>(htp->data);
-  if(downstream->get_response_header_key_prev()) {
-    downstream->append_last_response_header_key(data, len);
-  } else {
-    downstream->add_response_header(std::string(data, len), "");
-  }
-  return 0;
-}
-} // namespace
-
-namespace {
-int htp_hdr_valcb(http_parser *htp, const char *data, size_t len)
+int Downstream::on_read()
 {
-  Downstream *downstream;
-  downstream = reinterpret_cast<Downstream*>(htp->data);
-  if(downstream->get_response_header_key_prev()) {
-    downstream->set_last_response_header_value(std::string(data, len));
-  } else {
-    downstream->append_last_response_header_value(data, len);
-  }
-  return 0;
-}
-} // namespace
-
-namespace {
-int htp_bodycb(http_parser *htp, const char *data, size_t len)
-{
-  Downstream *downstream;
-  downstream = reinterpret_cast<Downstream*>(htp->data);
-
-  return downstream->get_upstream()->on_downstream_body
-    (downstream, reinterpret_cast<const uint8_t*>(data), len);
-}
-} // namespace
-
-namespace {
-int htp_msg_completecb(http_parser *htp)
-{
-  Downstream *downstream;
-  downstream = reinterpret_cast<Downstream*>(htp->data);
-
-  downstream->set_response_state(Downstream::MSG_COMPLETE);
-  return downstream->get_upstream()->on_downstream_body_complete(downstream);
-}
-} // namespace
-
-namespace {
-http_parser_settings htp_hooks = {
-  0, /*http_cb      on_message_begin;*/
-  0, /*http_data_cb on_url;*/
-  htp_hdr_keycb, /*http_data_cb on_header_field;*/
-  htp_hdr_valcb, /*http_data_cb on_header_value;*/
-  htp_hdrs_completecb, /*http_cb      on_headers_complete;*/
-  htp_bodycb, /*http_data_cb on_body;*/
-  htp_msg_completecb /*http_cb      on_message_complete;*/
-};
-} // namespace
-
-int Downstream::parse_http_response()
-{
-  bufferevent *bev = dconn_->get_bev();
-  evbuffer *input = bufferevent_get_input(bev);
-  unsigned char *mem = evbuffer_pullup(input, -1);
-
-  size_t nread = http_parser_execute(response_htp_, &htp_hooks,
-                                     reinterpret_cast<const char*>(mem),
-                                     evbuffer_get_length(input));
-
-  evbuffer_drain(input, nread);
-  http_errno htperr = HTTP_PARSER_ERRNO(response_htp_);
-  if(htperr == HPE_OK) {
-    return 0;
-  } else {
-    if(ENABLE_LOG) {
-      LOG(INFO) << "Downstream HTTP parser failure: "
-                << "(" << http_errno_name(htperr) << ") "
-                << http_errno_description(htperr);
-    }
-    return SHRPX_ERR_HTTP_PARSE;
-  }
+  return dconn_->on_read();
 }
 
 void Downstream::set_response_state(int state)

src/shrpx_downstream.h

@@ -35,10 +35,6 @@
 #include <event.h>
 #include <event2/bufferevent.h>
 
-extern "C" {
-#include "http-parser/http_parser.h"
-}
-
 #include "shrpx_io_control.h"
 
 namespace shrpx {
@@ -128,19 +124,21 @@ public:
   void set_chunked_response(bool f);
   bool get_response_connection_close() const;
   void set_response_connection_close(bool f);
-  int parse_http_response();
   void set_response_state(int state);
   int get_response_state() const;
   int init_response_body_buf();
   evbuffer* get_response_body_buf();
 
-  static const size_t DOWNSTREAM_OUTPUT_UPPER_THRES = 64*1024;
+  // Call this method when there is incoming data in downstream
+  // connection.
+  int on_read();
+
+  static const size_t OUTPUT_UPPER_THRES = 64*1024;
 private:
   Upstream *upstream_;
   DownstreamConnection *dconn_;
   int32_t stream_id_;
   int priority_;
-  IOControl ioctrl_;
   // stream ID in backend connection
   int32_t downstream_stream_id_;
 
@@ -163,7 +161,6 @@ private:
   bool response_connection_close_;
   Headers response_headers_;
   bool response_header_key_prev_;
-  http_parser *response_htp_;
   // This buffer is used to temporarily store downstream response
   // body. Spdylay reads data from this in the callback.
   evbuffer *response_body_buf_;

src/shrpx_downstream_connection.cc

@@ -25,96 +25,17 @@
 #include "shrpx_downstream_connection.h"
 
 #include "shrpx_client_handler.h"
-#include "shrpx_upstream.h"
 #include "shrpx_downstream.h"
-#include "shrpx_config.h"
-#include "shrpx_error.h"
 
 namespace shrpx {
 
 DownstreamConnection::DownstreamConnection(ClientHandler *client_handler)
   : client_handler_(client_handler),
-    bev_(0),
     downstream_(0)
 {}
 
 DownstreamConnection::~DownstreamConnection()
-{
-  if(bev_) {
-    bufferevent_disable(bev_, EV_READ | EV_WRITE);
-    bufferevent_free(bev_);
-  }
-  // Downstream and DownstreamConnection may be deleted
-  // asynchronously.
-  if(downstream_) {
-    downstream_->set_downstream_connection(0);
-  }
-}
-
-// When downstream request is issued, call this function to set read
-// timeout. We don't know when the request is completely received by
-// the downstream server. This function may be called before that
-// happens. Overall it does not cause problem for most of the time.
-// If the downstream server is too slow to recv/send, the connection
-// will be dropped by read timeout.
-void DownstreamConnection::start_waiting_response()
-{
-  if(bev_) {
-    bufferevent_set_timeouts(bev_,
-                             &get_config()->downstream_read_timeout,
-                             &get_config()->downstream_write_timeout);
-  }
-}
-
-namespace {
-// Gets called when DownstreamConnection is pooled in ClientHandler.
-void idle_eventcb(bufferevent *bev, short events, void *arg)
-{
-  DownstreamConnection *dconn = reinterpret_cast<DownstreamConnection*>(arg);
-  if(events & BEV_EVENT_CONNECTED) {
-    // Downstream was detached before connection established?
-    // This may be safe to be left.
-    if(ENABLE_LOG) {
-      LOG(INFO) << "Idle downstream connected?" << dconn;
-    }
-    return;
-  }
-  if(events & BEV_EVENT_EOF) {
-    if(ENABLE_LOG) {
-      LOG(INFO) << "Idle downstream connection EOF " << dconn;
-    }
-  } else if(events & BEV_EVENT_TIMEOUT) {
-    if(ENABLE_LOG) {
-      LOG(INFO) << "Idle downstream connection timeout " << dconn;
-    }
-  } else if(events & BEV_EVENT_ERROR) {
-    if(ENABLE_LOG) {
-      LOG(INFO) << "Idle downstream connection error " << dconn;
-    }
-  }
-  ClientHandler *client_handler = dconn->get_client_handler();
-  client_handler->remove_downstream_connection(dconn);
-  delete dconn;
-}
-} // namespace
-
-void DownstreamConnection::detach_downstream(Downstream *downstream)
-{
-  if(ENABLE_LOG) {
-    LOG(INFO) << "Detaching downstream connection " << this << " from "
-              << "downstream " << downstream;
-  }
-  downstream->set_downstream_connection(0);
-  downstream_ = 0;
-  bufferevent_enable(bev_, EV_READ);
-  bufferevent_setcb(bev_, 0, 0, idle_eventcb, this);
-  // On idle state, just enable read timeout. Normally idle downstream
-  // connection will get EOF from the downstream server and closed.
-  bufferevent_set_timeouts(bev_,
-                           &get_config()->downstream_idle_read_timeout,
-                           &get_config()->downstream_write_timeout);
-  client_handler_->pool_downstream_connection(this);
-}
+{}
 
 ClientHandler* DownstreamConnection::get_client_handler()
 {
@@ -126,9 +47,4 @@ Downstream* DownstreamConnection::get_downstream()
   return downstream_;
 }
 
-bufferevent* DownstreamConnection::get_bev()
-{
-  return bev_;
-}
-
 } // namespace shrpx

src/shrpx_downstream_connection.h

@@ -27,8 +27,7 @@
 
 #include "shrpx.h"
 
-#include <event.h>
-#include <event2/bufferevent.h>
+#include "shrpx_io_control.h"
 
 namespace shrpx {
 
@@ -40,21 +39,25 @@ public:
   DownstreamConnection(ClientHandler *client_handler);
   virtual ~DownstreamConnection();
   virtual int attach_downstream(Downstream *downstream) = 0;
-  void detach_downstream(Downstream *downstream);
-  bufferevent* get_bev();
+  virtual void detach_downstream(Downstream *downstream) = 0;
 
   virtual int push_request_headers() = 0;
   virtual int push_upload_data_chunk(const uint8_t *data, size_t datalen) = 0;
   virtual int end_upload_data() = 0;
 
-  virtual int on_connect() = 0;
+  virtual void pause_read(IOCtrlReason reason) = 0;
+  virtual bool resume_read(IOCtrlReason reason) = 0;
+  virtual void force_resume_read() = 0;
+
+  virtual bool get_output_buffer_full() = 0;
+
+  virtual int on_read() = 0;
+  virtual int on_write() = 0;
 
   ClientHandler* get_client_handler();
   Downstream* get_downstream();
-  void start_waiting_response();
 protected:
   ClientHandler *client_handler_;
-  bufferevent *bev_;
   Downstream *downstream_;
 };
 

src/shrpx_http_downstream_connection.cc

@@ -44,11 +44,25 @@ timeval max_timeout = { 86400, 0 };
 
 HttpDownstreamConnection::HttpDownstreamConnection
 (ClientHandler *client_handler)
-  : DownstreamConnection(client_handler)
+  : DownstreamConnection(client_handler),
+    bev_(0),
+    ioctrl_(0),
+    response_htp_(new http_parser())
 {}
 
 HttpDownstreamConnection::~HttpDownstreamConnection()
-{}
+{
+  delete response_htp_;
+  if(bev_) {
+    bufferevent_disable(bev_, EV_READ | EV_WRITE);
+    bufferevent_free(bev_);
+  }
+  // Downstream and DownstreamConnection may be deleted
+  // asynchronously.
+  if(downstream_) {
+    downstream_->set_downstream_connection(0);
+  }
+}
 
 int HttpDownstreamConnection::attach_downstream(Downstream *downstream)
 {
@@ -78,6 +92,12 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream)
   }
   downstream->set_downstream_connection(this);
   downstream_ = downstream;
+
+  ioctrl_.set_bev(bev_);
+
+  http_parser_init(response_htp_, HTTP_RESPONSE);
+  response_htp_->data = downstream_;
+
   bufferevent_setwatermark(bev_, EV_READ, 0, SHRPX_READ_WARTER_MARK);
   bufferevent_enable(bev_, EV_READ);
   bufferevent_setcb(bev_,
@@ -172,7 +192,17 @@ int HttpDownstreamConnection::push_request_headers()
   if(rv != 0) {
     return -1;
   }
-  start_waiting_response();
+
+  // When downstream request is issued, set read timeout. We don't
+  // know when the request is completely received by the downstream
+  // server. This function may be called before that happens. Overall
+  // it does not cause problem for most of the time.  If the
+  // downstream server is too slow to recv/send, the connection will
+  // be dropped by read timeout.
+  bufferevent_set_timeouts(bev_,
+                           &get_config()->downstream_read_timeout,
+                           &get_config()->downstream_write_timeout);
+
   return 0;
 }
 
@@ -223,7 +253,196 @@ int HttpDownstreamConnection::end_upload_data()
   return 0;
 }
 
-int HttpDownstreamConnection::on_connect()
+namespace {
+// Gets called when DownstreamConnection is pooled in ClientHandler.
+void idle_eventcb(bufferevent *bev, short events, void *arg)
+{
+  HttpDownstreamConnection *dconn;
+  dconn = reinterpret_cast<HttpDownstreamConnection*>(arg);
+  if(events & BEV_EVENT_CONNECTED) {
+    // Downstream was detached before connection established?
+    // This may be safe to be left.
+    if(ENABLE_LOG) {
+      LOG(INFO) << "Idle downstream connected?" << dconn;
+    }
+    return;
+  }
+  if(events & BEV_EVENT_EOF) {
+    if(ENABLE_LOG) {
+      LOG(INFO) << "Idle downstream connection EOF " << dconn;
+    }
+  } else if(events & BEV_EVENT_TIMEOUT) {
+    if(ENABLE_LOG) {
+      LOG(INFO) << "Idle downstream connection timeout " << dconn;
+    }
+  } else if(events & BEV_EVENT_ERROR) {
+    if(ENABLE_LOG) {
+      LOG(INFO) << "Idle downstream connection error " << dconn;
+    }
+  }
+  ClientHandler *client_handler = dconn->get_client_handler();
+  client_handler->remove_downstream_connection(dconn);
+  delete dconn;
+}
+} // namespace
+
+void HttpDownstreamConnection::detach_downstream(Downstream *downstream)
+{
+  if(ENABLE_LOG) {
+    LOG(INFO) << "Detaching downstream connection " << this << " from "
+              << "downstream " << downstream;
+  }
+  downstream->set_downstream_connection(0);
+  downstream_ = 0;
+  ioctrl_.force_resume_read();
+  bufferevent_enable(bev_, EV_READ);
+  bufferevent_setcb(bev_, 0, 0, idle_eventcb, this);
+  // On idle state, just enable read timeout. Normally idle downstream
+  // connection will get EOF from the downstream server and closed.
+  bufferevent_set_timeouts(bev_,
+                           &get_config()->downstream_idle_read_timeout,
+                           &get_config()->downstream_write_timeout);
+  client_handler_->pool_downstream_connection(this);
+}
+
+bufferevent* HttpDownstreamConnection::get_bev()
+{
+  return bev_;
+}
+
+void HttpDownstreamConnection::pause_read(IOCtrlReason reason)
+{
+  ioctrl_.pause_read(reason);
+}
+
+bool HttpDownstreamConnection::resume_read(IOCtrlReason reason)
+{
+  return ioctrl_.resume_read(reason);
+}
+
+void HttpDownstreamConnection::force_resume_read()
+{
+  ioctrl_.force_resume_read();
+}
+
+bool HttpDownstreamConnection::get_output_buffer_full()
+{
+  evbuffer *output = bufferevent_get_output(bev_);
+  return evbuffer_get_length(output) >= Downstream::OUTPUT_UPPER_THRES;
+}
+
+namespace {
+int htp_hdrs_completecb(http_parser *htp)
+{
+  Downstream *downstream;
+  downstream = reinterpret_cast<Downstream*>(htp->data);
+  downstream->set_response_http_status(htp->status_code);
+  downstream->set_response_major(htp->http_major);
+  downstream->set_response_minor(htp->http_minor);
+  downstream->set_response_connection_close(!http_should_keep_alive(htp));
+  downstream->set_response_state(Downstream::HEADER_COMPLETE);
+  if(downstream->get_upstream()->on_downstream_header_complete(downstream)
+     != 0) {
+    return -1;
+  }
+  unsigned int status = downstream->get_response_http_status();
+  // Ignore the response body. HEAD response may contain
+  // Content-Length or Transfer-Encoding: chunked.  Some server send
+  // 304 status code with nonzero Content-Length, but without response
+  // body. See
+  // http://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-20#section-3.3
+  return downstream->get_request_method() == "HEAD" ||
+    (100 <= status && status <= 199) || status == 204 ||
+    status == 304 ? 1 : 0;
+}
+} // namespace
+
+namespace {
+int htp_hdr_keycb(http_parser *htp, const char *data, size_t len)
+{
+  Downstream *downstream;
+  downstream = reinterpret_cast<Downstream*>(htp->data);
+  if(downstream->get_response_header_key_prev()) {
+    downstream->append_last_response_header_key(data, len);
+  } else {
+    downstream->add_response_header(std::string(data, len), "");
+  }
+  return 0;
+}
+} // namespace
+
+namespace {
+int htp_hdr_valcb(http_parser *htp, const char *data, size_t len)
+{
+  Downstream *downstream;
+  downstream = reinterpret_cast<Downstream*>(htp->data);
+  if(downstream->get_response_header_key_prev()) {
+    downstream->set_last_response_header_value(std::string(data, len));
+  } else {
+    downstream->append_last_response_header_value(data, len);
+  }
+  return 0;
+}
+} // namespace
+
+namespace {
+int htp_bodycb(http_parser *htp, const char *data, size_t len)
+{
+  Downstream *downstream;
+  downstream = reinterpret_cast<Downstream*>(htp->data);
+
+  return downstream->get_upstream()->on_downstream_body
+    (downstream, reinterpret_cast<const uint8_t*>(data), len);
+}
+} // namespace
+
+namespace {
+int htp_msg_completecb(http_parser *htp)
+{
+  Downstream *downstream;
+  downstream = reinterpret_cast<Downstream*>(htp->data);
+
+  downstream->set_response_state(Downstream::MSG_COMPLETE);
+  return downstream->get_upstream()->on_downstream_body_complete(downstream);
+}
+} // namespace
+
+namespace {
+http_parser_settings htp_hooks = {
+  0, /*http_cb      on_message_begin;*/
+  0, /*http_data_cb on_url;*/
+  htp_hdr_keycb, /*http_data_cb on_header_field;*/
+  htp_hdr_valcb, /*http_data_cb on_header_value;*/
+  htp_hdrs_completecb, /*http_cb      on_headers_complete;*/
+  htp_bodycb, /*http_data_cb on_body;*/
+  htp_msg_completecb /*http_cb      on_message_complete;*/
+};
+} // namespace
+
+int HttpDownstreamConnection::on_read()
+{
+  evbuffer *input = bufferevent_get_input(bev_);
+  unsigned char *mem = evbuffer_pullup(input, -1);
+
+  size_t nread = http_parser_execute(response_htp_, &htp_hooks,
+                                     reinterpret_cast<const char*>(mem),
+                                     evbuffer_get_length(input));
+
+  evbuffer_drain(input, nread);
+  http_errno htperr = HTTP_PARSER_ERRNO(response_htp_);
+  if(htperr == HPE_OK) {
+    return 0;
+  } else {
+    if(ENABLE_LOG) {
+      LOG(INFO) << "Downstream HTTP parser failure: "
+                << "(" << http_errno_name(htperr) << ") "
+                << http_errno_description(htperr);
+    }
+    return SHRPX_ERR_HTTP_PARSE;
+  }
+}
+
+int HttpDownstreamConnection::on_write()
 {
   return 0;
 }

src/shrpx_http_downstream_connection.h

@@ -27,7 +27,13 @@
 
 #include "shrpx.h"
 
+#include <event.h>
+#include <event2/bufferevent.h>
+
+#include "http-parser/http_parser.h"
+
 #include "shrpx_downstream_connection.h"
+#include "shrpx_io_control.h"
 
 namespace shrpx {
 
@@ -36,12 +42,26 @@ public:
   HttpDownstreamConnection(ClientHandler *client_handler);
   virtual ~HttpDownstreamConnection();
   virtual int attach_downstream(Downstream *downstream);
+  virtual void detach_downstream(Downstream *downstream);
 
   virtual int push_request_headers();
   virtual int push_upload_data_chunk(const uint8_t *data, size_t datalen);
   virtual int end_upload_data();
 
-  virtual int on_connect();
+  virtual void pause_read(IOCtrlReason reason);
+  virtual bool resume_read(IOCtrlReason reason);
+  virtual void force_resume_read();
+
+  virtual bool get_output_buffer_full();
+
+  virtual int on_read();
+  virtual int on_write();
+
+  bufferevent* get_bev();
+private:
+  bufferevent *bev_;
+  IOControl ioctrl_;
+  http_parser *response_htp_;
 };
 
 } // namespace shrpx

src/shrpx_https_upstream.cc

@@ -317,13 +317,15 @@ void HttpsUpstream::pause_read(IOCtrlReason reason)
   ioctrl_.pause_read(reason);
 }
 
-void HttpsUpstream::resume_read(IOCtrlReason reason)
+int HttpsUpstream::resume_read(IOCtrlReason reason)
 {
   if(ioctrl_.resume_read(reason)) {
     // Process remaining data in input buffer here because these bytes
     // are not notified by readcb until new data arrive.
     http_parser_pause(htp_, 0);
-    on_read();
+    return on_read();
+  } else {
+    return 0;
   }
 }
 
@@ -335,12 +337,10 @@ void https_downstream_readcb(bufferevent *bev, void *ptr)
   HttpsUpstream *upstream;
   upstream = static_cast<HttpsUpstream*>(downstream->get_upstream());
   int rv;
-  if(get_config()->client_mode) {
-    rv = reinterpret_cast<SpdyDownstreamConnection*>(dconn)->on_read();
-  } else {
-    rv = downstream->parse_http_response();
-  }
-  if(rv == 0) {
+  rv = downstream->on_read();
+  if(downstream->get_response_state() == Downstream::MSG_RESET) {
+    delete upstream->get_client_handler();
+  } else if(rv == 0) {
     if(downstream->get_response_state() == Downstream::MSG_COMPLETE) {
       if(downstream->get_response_connection_close()) {
         // Connection close
@@ -379,8 +379,6 @@ void https_downstream_readcb(bufferevent *bev, void *ptr)
       // We already sent HTTP response headers to upstream
       // client. Just close the upstream connection.
       delete upstream->get_client_handler();
-    } else if(downstream->get_response_state() == Downstream::MSG_RESET) {
-      delete upstream->get_client_handler();
     } else {
       // We did not sent any HTTP response, so sent error
       // response. Cannot reuse downstream connection in this case.
@@ -406,14 +404,6 @@ void https_downstream_writecb(bufferevent *bev, void *ptr)
   HttpsUpstream *upstream;
   upstream = static_cast<HttpsUpstream*>(downstream->get_upstream());
   upstream->resume_read(SHRPX_NO_BUFFER);
-  if(get_config()->client_mode) {
-    int rv;
-    rv = reinterpret_cast<SpdyDownstreamConnection*>(dconn)->on_write();
-    if(rv != 0) {
-      delete upstream->get_client_handler();
-      return;
-    }
-  }
 }
 } // namespace
 
@@ -429,11 +419,6 @@ void https_downstream_eventcb(bufferevent *bev, short events, void *ptr)
       LOG(INFO) << "Downstream connection established. downstream "
                 << downstream;
     }
-    if(dconn->on_connect() != 0) {
-      // TODO Return error status 502
-      delete upstream->get_client_handler();
-      return;
-    }
   } else if(events & BEV_EVENT_EOF) {
     if(ENABLE_LOG) {
       LOG(INFO) << "Downstream EOF. stream_id="

src/shrpx_https_upstream.h

@@ -57,7 +57,7 @@ public:
   int error_reply(int status_code);
 
   virtual void pause_read(IOCtrlReason reason);
-  virtual void resume_read(IOCtrlReason reason);
+  virtual int resume_read(IOCtrlReason reason);
 
   virtual int on_downstream_header_complete(Downstream *downstream);
   virtual int on_downstream_body(Downstream *downstream,

src/shrpx_listen_handler.cc

@@ -36,6 +36,7 @@
 #include "shrpx_ssl.h"
 #include "shrpx_worker.h"
 #include "shrpx_config.h"
+#include "shrpx_spdy_session.h"
 
 namespace shrpx {
 
@@ -45,7 +46,8 @@ ListenHandler::ListenHandler(event_base *evbase)
              ssl::create_ssl_client_context() : ssl::create_ssl_context()),
     worker_round_robin_cnt_(0),
     workers_(0),
-    num_worker_(0)
+    num_worker_(0),
+    spdy_(0)
 {}
 
 ListenHandler::~ListenHandler()
@@ -93,8 +95,11 @@ int ListenHandler::accept_connection(evutil_socket_t fd,
     LOG(INFO) << "<listener> Accepted connection. fd=" << fd;
   }
   if(num_worker_ == 0) {
-    /*ClientHandler* client = */
-    ssl::accept_ssl_connection(evbase_, ssl_ctx_, fd, addr, addrlen);
+    ClientHandler* client =
+      ssl::accept_ssl_connection(evbase_, ssl_ctx_, fd, addr, addrlen);
+    if(get_config()->client_mode) {
+      client->set_spdy_session(spdy_);
+    }
   } else {
     size_t idx = worker_round_robin_cnt_ % num_worker_;
     ++worker_round_robin_cnt_;
@@ -114,4 +119,12 @@ event_base* ListenHandler::get_evbase() const
   return evbase_;
 }
 
+int ListenHandler::create_spdy_session()
+{
+  int rv;
+  spdy_ = new SpdySession(evbase_, ssl_ctx_);
+  rv = spdy_->init_notification();
+  return rv;
+}
+
 } // namespace shrpx

src/shrpx_listen_handler.h

@@ -42,6 +42,8 @@ struct WorkerInfo {
   bufferevent *bev;
 };
 
+class SpdySession;
+
 class ListenHandler {
 public:
   ListenHandler(event_base *evbase);
@@ -49,14 +51,18 @@ public:
   int accept_connection(evutil_socket_t fd, sockaddr *addr, int addrlen);
   void create_worker_thread(size_t num);
   event_base* get_evbase() const;
+  int create_spdy_session();
 private:
   event_base *evbase_;
   // In client-mode, this is for backend SPDY connection. Otherwise,
-  // frontend SSL/TLS connection.
+  // for frontend.
   SSL_CTX *ssl_ctx_;
   unsigned int worker_round_robin_cnt_;
   WorkerInfo *workers_;
   size_t num_worker_;
+  // Shared SPDY session. NULL if not client mode or
+  // multi-threaded. In multi-threaded case, see shrpx_worker.cc.
+  SpdySession *spdy_;
 };
 
 } // namespace shrpx

src/shrpx_spdy_downstream_connection.h

@@ -35,28 +35,39 @@
 
 namespace shrpx {
 
+struct StreamData;
+class SpdySession;
+
 class SpdyDownstreamConnection : public DownstreamConnection {
 public:
   SpdyDownstreamConnection(ClientHandler *client_handler);
   virtual ~SpdyDownstreamConnection();
   virtual int attach_downstream(Downstream *downstream);
+  virtual void detach_downstream(Downstream *downstream);
 
   virtual int push_request_headers();
   virtual int push_upload_data_chunk(const uint8_t *data, size_t datalen);
   virtual int end_upload_data();
 
-  virtual int on_connect();
+  virtual void pause_read(IOCtrlReason reason) {}
+  virtual bool resume_read(IOCtrlReason reason) { return true; }
+  virtual void force_resume_read() {}
+
+  virtual bool get_output_buffer_full();
 
-  int on_read();
-  int on_write();
+  virtual int on_read();
+  virtual int on_write();
   int send();
 
   int init_request_body_buf();
   evbuffer* get_request_body_buf() const;
+
+  void attach_stream_data(StreamData *sd);
+  StreamData* detach_stream_data();
 private:
-  SSL *ssl_;
-  spdylay_session *session_;
+  SpdySession *spdy_;
   evbuffer *request_body_buf_;
+  StreamData *sd_;
 };
 
 } // namespace shrpx

src/shrpx_spdy_session.h

+/*
+ * Spdylay - SPDY Library
+ *
+ * Copyright (c) 2012 Tatsuhiro Tsujikawa
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+#ifndef SHRPX_SPDY_SESSION_H
+#define SHRPX_SPDY_SESSION_H
+
+#include "shrpx.h"
+
+#include <set>
+
+#include <openssl/ssl.h>
+
+#include <event.h>
+#include <event2/bufferevent.h>
+
+#include <spdylay/spdylay.h>
+
+namespace shrpx {
+
+class SpdyDownstreamConnection;
+
+struct StreamData {
+  SpdyDownstreamConnection *dconn;
+};
+
+class SpdySession {
+public:
+  SpdySession(event_base *evbase, SSL_CTX *ssl_ctx);
+  ~SpdySession();
+
+  int init_notification();
+
+  int disconnect();
+  int initiate_connection();
+  void connected();
+
+  void add_downstream_connection(SpdyDownstreamConnection *dconn);
+  void remove_downstream_connection(SpdyDownstreamConnection *dconn);
+
+  void remove_stream_data(StreamData *sd);
+
+  int submit_request(SpdyDownstreamConnection *dconn,
+                     uint8_t pri, const char **nv,
+                     const spdylay_data_provider *data_prd);
+
+  int submit_rst_stream(SpdyDownstreamConnection *docnn,
+                        int32_t stream_id, uint32_t status_code);
+
+  int resume_data(SpdyDownstreamConnection *dconn);
+
+  int on_connect();
+
+  int on_read();
+  int on_write();
+  int send();
+
+  void clear_notify();
+  void notify();
+
+  bufferevent* get_bev() const;
+
+  int get_state() const;
+
+  enum {
+    DISCONNECTED,
+    CONNECTING,
+    CONNECTED
+  };
+private:
+  event_base *evbase_;
+  SSL_CTX *ssl_ctx_;
+  SSL *ssl_;
+  spdylay_session *session_;
+  bufferevent *bev_;
+  std::set<SpdyDownstreamConnection*> dconns_;
+  std::set<StreamData*> streams_;
+  int state_;
+  bool notified_;
+  bufferevent *wrbev_;
+  bufferevent *rdbev_;
+};
+
+} // namespace shrpx
+
+#endif // SHRPX_SPDY_SESSION_H

src/shrpx_spdy_upstream.cc

@@ -460,7 +460,7 @@ void spdy_downstream_readcb(bufferevent *bev, void *ptr)
     delete downstream;
     return;
   }
-  int rv = downstream->parse_http_response();
+  int rv = downstream->on_read();
   if(rv != 0) {
     if(ENABLE_LOG) {
       LOG(INFO) << "Downstream HTTP parser failure";
@@ -846,7 +846,9 @@ int32_t SpdyUpstream::get_initial_window_size() const
 void SpdyUpstream::pause_read(IOCtrlReason reason)
 {}
 
-void SpdyUpstream::resume_read(IOCtrlReason reason)
-{}
+int SpdyUpstream::resume_read(IOCtrlReason reason)
+{
+  return 0;
+}
 
 } // namespace shrpx

src/shrpx_spdy_upstream.h

@@ -59,7 +59,7 @@ public:
   int error_reply(Downstream *downstream, int status_code);
 
   virtual void pause_read(IOCtrlReason reason);
-  virtual void resume_read(IOCtrlReason reason);
+  virtual int resume_read(IOCtrlReason reason);
 
   virtual int on_downstream_header_complete(Downstream *downstream);
   virtual int on_downstream_body(Downstream *downstream,

src/shrpx_ssl.cc

@@ -221,9 +221,6 @@ ClientHandler* accept_ssl_connection(event_base *evbase, SSL_CTX *ssl_ctx,
          BUFFEREVENT_SSL_ACCEPTING, BEV_OPT_DEFER_CALLBACKS);
     }
     ClientHandler *client_handler = new ClientHandler(bev, fd, ssl, host);
-    if(get_config()->client_mode) {
-      client_handler->set_ssl_client_ctx(ssl_ctx);
-    }
     return client_handler;
   } else {
     LOG(ERROR) << "getnameinfo() failed: " << gai_strerror(rv);

src/shrpx_thread_event_receiver.cc

@@ -29,11 +29,13 @@
 #include "shrpx_ssl.h"
 #include "shrpx_log.h"
 #include "shrpx_client_handler.h"
+#include "shrpx_spdy_session.h"
 
 namespace shrpx {
 
-ThreadEventReceiver::ThreadEventReceiver(SSL_CTX *ssl_ctx)
-  : ssl_ctx_(ssl_ctx)
+ThreadEventReceiver::ThreadEventReceiver(SSL_CTX *ssl_ctx, SpdySession *spdy)
+  : ssl_ctx_(ssl_ctx),
+    spdy_(spdy)
 {}
 
 ThreadEventReceiver::~ThreadEventReceiver()
@@ -56,6 +58,7 @@ void ThreadEventReceiver::on_read(bufferevent *bev)
                                                 &wev.client_addr.sa,
                                                 wev.client_addrlen);
     if(client_handler) {
+      client_handler->set_spdy_session(spdy_);
       if(ENABLE_LOG) {
         LOG(INFO) << "ClientHandler " << client_handler << " created";
       }

src/shrpx_thread_event_receiver.h

@@ -35,6 +35,8 @@
 
 namespace shrpx {
 
+class SpdySession;
+
 struct WorkerEvent {
   evutil_socket_t client_fd;
   sockaddr_union client_addr;
@@ -43,11 +45,14 @@ struct WorkerEvent {
 
 class ThreadEventReceiver {
 public:
-  ThreadEventReceiver(SSL_CTX *ssl_ctx);
+  ThreadEventReceiver(SSL_CTX *ssl_ctx, SpdySession *spdy);
   ~ThreadEventReceiver();
   void on_read(bufferevent *bev);
 private:
   SSL_CTX *ssl_ctx_;
+  // Shared SPDY session for each thread. NULL if not client mode. Not
+  // deleted by this object.
+  SpdySession *spdy_;
 };
 
 } // namespace shrpx

src/shrpx_upstream.h

@@ -53,7 +53,7 @@ public:
   virtual int on_downstream_body_complete(Downstream *downstream) = 0;
 
   virtual void pause_read(IOCtrlReason reason) = 0;
-  virtual void resume_read(IOCtrlReason reason) = 0;
+  virtual int resume_read(IOCtrlReason reason) = 0;
 };
 
 } // namespace shrpx

src/shrpx_worker.cc

@@ -33,6 +33,7 @@
 #include "shrpx_ssl.h"
 #include "shrpx_thread_event_receiver.h"
 #include "shrpx_log.h"
+#include "shrpx_spdy_session.h"
 
 namespace shrpx {
 
@@ -72,7 +73,14 @@ void Worker::run()
   event_base *evbase = event_base_new();
   bufferevent *bev = bufferevent_socket_new(evbase, fd_,
                                             BEV_OPT_DEFER_CALLBACKS);
-  ThreadEventReceiver *receiver = new ThreadEventReceiver(ssl_ctx_);
+  SpdySession *spdy = 0;
+  if(get_config()->client_mode) {
+    spdy = new SpdySession(evbase, ssl_ctx_);
+    if(spdy->init_notification() == -1) {
+      DIE();
+    }
+  }
+  ThreadEventReceiver *receiver = new ThreadEventReceiver(ssl_ctx_, spdy);
   bufferevent_enable(bev, EV_READ);
   bufferevent_setcb(bev, readcb, 0, eventcb, receiver);