Merge branch 'nghttpx-backend-h1-tls'

doc/sources/nghttpx-howto.rst

@@ -14,9 +14,10 @@ If nghttpx is invoked without any ``-s``, ``-p`` and ``--client``, it
 operates in default mode.  In this mode, nghttpx frontend listens for
 HTTP/2 requests and translates them to HTTP/1 requests.  Thus it works
 as reverse proxy (gateway) for HTTP/2 clients to HTTP/1 web server.
-HTTP/1 requests are also supported in frontend as a fallback.  If
-nghttpx is linked with spdylay library and frontend connection is
-SSL/TLS, the frontend also supports SPDY protocol.
+This is also known as "HTTP/2 router".  HTTP/1 requests are also
+supported in frontend as a fallback.  If nghttpx is linked with
+spdylay library and frontend connection is SSL/TLS, the frontend also
+supports SPDY protocol.
 
 By default, this mode's frontend connection is encrypted using
 SSL/TLS.  So server's private key and certificate must be supplied to
@@ -30,6 +31,10 @@ available on the frontend and a HTTP/1 connection can be upgraded to
 HTTP/2 using HTTP Upgrade.  Starting HTTP/2 connection by sending
 HTTP/2 connection preface is also supported.
 
+By default, backend HTTP/1 connections are not encrypted.  To enable
+TLS on HTTP/1 backend connections, use ``--backend-http1-tls`` option.
+This applies to all mode whose backend connections are HTTP/1.
+
 The backend is supposed to be HTTP/1 Web server.  For example, to make
 nghttpx listen to encrypted HTTP/2 requests at port 8443, and a
 backend HTTP/1 web server is configured to listen to HTTP/1 request at

gennghttpxfun.py

@@ -112,7 +112,9 @@ OPTIONS = [
     "max-request-header-fields",
     "header-field-buffer",
     "max-header-fields",
-    "no-http2-cipher-black-list"
+    "no-http2-cipher-black-list",
+    "backend-http1-tls",
+    "backend-tls-session-cache-per-worker"
 ]
 
 LOGVARS = [

src/shrpx.cc

@@ -1046,8 +1046,6 @@ void fill_default_config() {
   auto &tlsconf = mod_config()->tls;
   {
     auto &ticketconf = tlsconf.ticket;
-    ticketconf.cipher = EVP_aes_128_cbc();
-
     {
       auto &memcachedconf = ticketconf.memcached;
       memcachedconf.max_retry = 3;
@@ -1055,19 +1053,26 @@ void fill_default_config() {
       memcachedconf.interval = 10_min;
     }
 
+    ticketconf.cipher = EVP_aes_128_cbc();
+  }
+
+  {
     auto &ocspconf = tlsconf.ocsp;
     // ocsp update interval = 14400 secs = 4 hours, borrowed from h2o
     ocspconf.update_interval = 4_h;
     ocspconf.fetch_ocsp_response_file =
         strcopy(PKGDATADIR "/fetch-ocsp-response");
+  }
 
+  {
     auto &dyn_recconf = tlsconf.dyn_rec;
     dyn_recconf.warmup_threshold = 1_m;
     dyn_recconf.idle_timeout = 1_s;
-
-    tlsconf.session_timeout = std::chrono::hours(12);
   }
 
+  tlsconf.session_timeout = std::chrono::hours(12);
+  tlsconf.downstream_session_cache_per_worker = 10000;
+
   auto &httpconf = mod_config()->http;
   httpconf.server_name = "nghttpx nghttp2/" NGHTTP2_VERSION;
   httpconf.no_host_rewrite = true;
@@ -1277,6 +1282,16 @@ Connections:
               --backend-write-timeout options.
   --accept-proxy-protocol
               Accept PROXY protocol version 1 on frontend connection.
+  --backend-no-tls
+              Disable  SSL/TLS  on  backend connections.   For  HTTP/2
+              backend  connections, TLS  is enabled  by default.   For
+              HTTP/1 backend connections, TLS  is disabled by default,
+              and can  be enabled  by --backend-http1-tls  option.  If
+              both  --backend-no-tls  and --backend-http1-tls  options
+              are used, --backend-no-tls has the precedence.
+  --backend-http1-tls
+              Enable SSL/TLS on backend  HTTP/1 connections.  See also
+              --backend-no-tls option.
 
 Performance:
   -n, --workers=<N>
@@ -1426,16 +1441,14 @@ SSL/TLS:
               Set allowed  cipher list.  The  format of the  string is
               described in OpenSSL ciphers(1).
   -k, --insecure
-              Don't  verify   backend  server's  certificate   if  -p,
-              --client    or    --http2-bridge     are    given    and
-              --backend-no-tls is not given.
+              Don't  verify backend  server's  certificate  if TLS  is
+              enabled for backend connections.
   --cacert=<PATH>
-              Set path to trusted CA  certificate file if -p, --client
-              or --http2-bridge are given  and --backend-no-tls is not
-              given.  The file must be  in PEM format.  It can contain
-              multiple  certificates.    If  the  linked   OpenSSL  is
-              configured to  load system  wide certificates,  they are
-              loaded at startup regardless of this option.
+              Set path to trusted CA  certificate file used in backend
+              TLS connections.   The file must  be in PEM  format.  It
+              can  contain  multiple   certificates.   If  the  linked
+              OpenSSL is configured to  load system wide certificates,
+              they are loaded at startup regardless of this option.
   --private-key-passwd-file=<PATH>
               Path  to file  that contains  password for  the server's
               private key.   If none is  given and the private  key is
@@ -1575,6 +1588,11 @@ SSL/TLS:
               Allow black  listed cipher  suite on  HTTP/2 connection.
               See  https://tools.ietf.org/html/rfc7540#appendix-A  for
               the complete HTTP/2 cipher suites black list.
+  --backend-tls-session-cache-per-worker=<N>
+              Set  the maximum  number  of backend  TLS session  cache
+              stored per worker.
+              Default: )"
+      << get_config()->tls.downstream_session_cache_per_worker << R"(
 
 HTTP/2 and SPDY:
   -c, --http2-max-concurrent-streams=<N>
@@ -1603,8 +1621,6 @@ HTTP/2 and SPDY:
               connection to 2**<N>-1.
               Default: )"
       << get_config()->http2.downstream.connection_window_bits << R"(
-  --backend-no-tls
-              Disable SSL/TLS on backend connections.
   --http2-no-cookie-crumbling
               Don't crumble cookie header field.
   --padding=<N>
@@ -2029,6 +2045,10 @@ void process_options(
     downstreamconf.proto = PROTO_HTTP;
   }
 
+  if (downstreamconf.proto == PROTO_HTTP && !downstreamconf.http1_tls) {
+    downstreamconf.no_tls = true;
+  }
+
   if (!upstreamconf.no_tls &&
       (!tlsconf.private_key_file || !tlsconf.cert_file)) {
     print_usage(std::cerr);
@@ -2377,6 +2397,9 @@ int main(int argc, char **argv) {
         {SHRPX_OPT_NO_HTTP2_CIPHER_BLACK_LIST, no_argument, &flag, 103},
         {SHRPX_OPT_REQUEST_HEADER_FIELD_BUFFER, required_argument, &flag, 104},
         {SHRPX_OPT_MAX_REQUEST_HEADER_FIELDS, required_argument, &flag, 105},
+        {SHRPX_OPT_BACKEND_HTTP1_TLS, no_argument, &flag, 106},
+        {SHRPX_OPT_BACKEND_TLS_SESSION_CACHE_PER_WORKER, required_argument,
+         &flag, 107},
         {nullptr, 0, nullptr, 0}};
 
     int option_index = 0;
@@ -2826,6 +2849,15 @@ int main(int argc, char **argv) {
         // --max-request-header-fields
         cmdcfgs.emplace_back(SHRPX_OPT_MAX_REQUEST_HEADER_FIELDS, optarg);
         break;
+      case 106:
+        // --backend-http1-tls
+        cmdcfgs.emplace_back(SHRPX_OPT_BACKEND_HTTP1_TLS, "yes");
+        break;
+      case 107:
+        // --backend-tls-session-cache-per-worker
+        cmdcfgs.emplace_back(SHRPX_OPT_BACKEND_TLS_SESSION_CACHE_PER_WORKER,
+                             optarg);
+        break;
       default:
         break;
       }

src/shrpx_client_handler.cc

@@ -722,8 +722,8 @@ ClientHandler::get_downstream_connection(Downstream *downstream) {
       }
       dconn = make_unique<Http2DownstreamConnection>(dconn_pool, http2session);
     } else {
-      dconn =
-          make_unique<HttpDownstreamConnection>(dconn_pool, group, conn_.loop);
+      dconn = make_unique<HttpDownstreamConnection>(dconn_pool, group,
+                                                    conn_.loop, worker_);
     }
     dconn->set_client_handler(this);
     return dconn;

src/shrpx_config.cc

@@ -676,6 +676,7 @@ enum {
   SHRPX_OPTID_BACKEND_HTTP_PROXY_URI,
   SHRPX_OPTID_BACKEND_HTTP1_CONNECTIONS_PER_FRONTEND,
   SHRPX_OPTID_BACKEND_HTTP1_CONNECTIONS_PER_HOST,
+  SHRPX_OPTID_BACKEND_HTTP1_TLS,
   SHRPX_OPTID_BACKEND_HTTP2_CONNECTION_WINDOW_BITS,
   SHRPX_OPTID_BACKEND_HTTP2_CONNECTIONS_PER_WORKER,
   SHRPX_OPTID_BACKEND_HTTP2_WINDOW_BITS,
@@ -686,6 +687,7 @@ enum {
   SHRPX_OPTID_BACKEND_READ_TIMEOUT,
   SHRPX_OPTID_BACKEND_REQUEST_BUFFER,
   SHRPX_OPTID_BACKEND_RESPONSE_BUFFER,
+  SHRPX_OPTID_BACKEND_TLS_SESSION_CACHE_PER_WORKER,
   SHRPX_OPTID_BACKEND_TLS_SNI_FIELD,
   SHRPX_OPTID_BACKEND_WRITE_TIMEOUT,
   SHRPX_OPTID_BACKLOG,
@@ -1077,6 +1079,9 @@ int option_lookup_token(const char *name, size_t namelen) {
       }
       break;
     case 's':
+      if (util::strieq_l("backend-http1-tl", name, 16)) {
+        return SHRPX_OPTID_BACKEND_HTTP1_TLS;
+      }
       if (util::strieq_l("max-header-field", name, 16)) {
         return SHRPX_OPTID_MAX_HEADER_FIELDS;
       }
@@ -1378,6 +1383,9 @@ int option_lookup_token(const char *name, size_t namelen) {
       if (util::strieq_l("backend-http2-connections-per-worke", name, 35)) {
         return SHRPX_OPTID_BACKEND_HTTP2_CONNECTIONS_PER_WORKER;
       }
+      if (util::strieq_l("backend-tls-session-cache-per-worke", name, 35)) {
+        return SHRPX_OPTID_BACKEND_TLS_SESSION_CACHE_PER_WORKER;
+      }
       break;
     case 's':
       if (util::strieq_l("backend-http2-connection-window-bit", name, 35)) {
@@ -2214,6 +2222,13 @@ int parse_config(const char *opt, const char *optarg,
     mod_config()->tls.no_http2_cipher_black_list = util::strieq(optarg, "yes");
 
     return 0;
+  case SHRPX_OPTID_BACKEND_HTTP1_TLS:
+    mod_config()->conn.downstream.http1_tls = util::strieq(optarg, "yes");
+
+    return 0;
+  case SHRPX_OPTID_BACKEND_TLS_SESSION_CACHE_PER_WORKER:
+    return parse_uint(&mod_config()->tls.downstream_session_cache_per_worker,
+                      opt, optarg);
   case SHRPX_OPTID_CONF:
     LOG(WARN) << "conf: ignored";
 

src/shrpx_config.h

@@ -206,6 +206,9 @@ constexpr char SHRPX_OPT_MAX_RESPONSE_HEADER_FIELDS[] =
     "max-response-header-fields";
 constexpr char SHRPX_OPT_NO_HTTP2_CIPHER_BLACK_LIST[] =
     "no-http2-cipher-black-list";
+constexpr char SHRPX_OPT_BACKEND_HTTP1_TLS[] = "backend-http1-tls";
+constexpr char SHRPX_OPT_BACKEND_TLS_SESSION_CACHE_PER_WORKER[] =
+    "backend-tls-session-cache-per-worker";
 
 constexpr size_t SHRPX_OBFUSCATED_NODE_LENGTH = 8;
 
@@ -390,6 +393,7 @@ struct TLSConfig {
   std::vector<std::string> npn_list;
   // list of supported SSL/TLS protocol strings.
   std::vector<std::string> tls_proto_list;
+  size_t downstream_session_cache_per_worker;
   // Bit mask to disable SSL/TLS protocol versions.  This will be
   // passed to SSL_CTX_set_options().
   long int tls_proto_mask;
@@ -534,6 +538,7 @@ struct ConnectionConfig {
     // downstream protocol; this will be determined by given options.
     shrpx_proto proto;
     bool no_tls;
+    bool http1_tls;
     // true if IPv4 only; ipv4 and ipv6 are mutually exclusive; and
     // (ipv4 && ipv6) must be false.
     bool ipv4;

src/shrpx_http_downstream_connection.h

@@ -36,11 +36,12 @@
 namespace shrpx {
 
 class DownstreamConnectionPool;
+class Worker;
 
 class HttpDownstreamConnection : public DownstreamConnection {
 public:
   HttpDownstreamConnection(DownstreamConnectionPool *dconn_pool, size_t group,
-                           struct ev_loop *loop);
+                           struct ev_loop *loop, Worker *worker);
   virtual ~HttpDownstreamConnection();
   virtual int attach_downstream(Downstream *downstream);
   virtual void detach_downstream(Downstream *downstream);
@@ -61,17 +62,30 @@ public:
 
   virtual bool poolable() const { return true; }
 
-  int on_connect();
+  int read_clear();
+  int write_clear();
+  int read_tls();
+  int write_tls();
+
+  int process_input(const uint8_t *data, size_t datalen);
+  int tls_handshake();
+
+  int connected();
   void signal_write();
 
+  int noop();
+
 private:
   Connection conn_;
+  std::function<int(HttpDownstreamConnection &)> do_read_, do_write_;
+  Worker *worker_;
+  // nullptr if TLS is not used.
+  SSL_CTX *ssl_ctx_;
   IOControl ioctrl_;
   http_parser response_htp_;
   size_t group_;
   // index of get_config()->downstream_addrs this object is using
   size_t addr_idx_;
-  bool connected_;
 };
 
 } // namespace shrpx

src/shrpx_ssl.cc

@@ -628,9 +628,9 @@ SSL_CTX *create_ssl_context(const char *private_key_file, const char *cert_file
 }
 
 namespace {
-int select_next_proto_cb(SSL *ssl, unsigned char **out, unsigned char *outlen,
-                         const unsigned char *in, unsigned int inlen,
-                         void *arg) {
+int select_h2_next_proto_cb(SSL *ssl, unsigned char **out,
+                            unsigned char *outlen, const unsigned char *in,
+                            unsigned int inlen, void *arg) {
   if (!util::select_h2(const_cast<const unsigned char **>(out), outlen, in,
                        inlen)) {
     return SSL_TLSEXT_ERR_NOACK;
@@ -640,6 +640,24 @@ int select_next_proto_cb(SSL *ssl, unsigned char **out, unsigned char *outlen,
 }
 } // namespace
 
+namespace {
+int select_h1_next_proto_cb(SSL *ssl, unsigned char **out,
+                            unsigned char *outlen, const unsigned char *in,
+                            unsigned int inlen, void *arg) {
+  auto end = in + inlen;
+  for (; in < end;) {
+    if (util::streq_l(NGHTTP2_H1_1_ALPN, in, in[0] + 1)) {
+      *out = const_cast<unsigned char *>(in) + 1;
+      *outlen = in[0];
+      return SSL_TLSEXT_ERR_OK;
+    }
+    in += in[0] + 1;
+  }
+
+  return SSL_TLSEXT_ERR_NOACK;
+}
+} // namespace
+
 SSL_CTX *create_ssl_client_context(
 #ifdef HAVE_NEVERBLEED
     neverbleed_t *nb
@@ -722,15 +740,29 @@ SSL_CTX *create_ssl_client_context(
       DIE();
     }
   }
-  // NPN selection callback
-  SSL_CTX_set_next_proto_select_cb(ssl_ctx, select_next_proto_cb, nullptr);
+
+  auto &downstreamconf = get_config()->conn.downstream;
+
+  if (downstreamconf.proto == PROTO_HTTP2) {
+    // NPN selection callback
+    SSL_CTX_set_next_proto_select_cb(ssl_ctx, select_h2_next_proto_cb, nullptr);
 
 #if OPENSSL_VERSION_NUMBER >= 0x10002000L
-  // ALPN advertisement; We only advertise HTTP/2
-  auto proto_list = util::get_default_alpn();
+    // ALPN advertisement; We only advertise HTTP/2
+    auto proto_list = util::get_default_alpn();
 
-  SSL_CTX_set_alpn_protos(ssl_ctx, proto_list.data(), proto_list.size());
+    SSL_CTX_set_alpn_protos(ssl_ctx, proto_list.data(), proto_list.size());
 #endif // OPENSSL_VERSION_NUMBER >= 0x10002000L
+  } else {
+    // NPN selection callback
+    SSL_CTX_set_next_proto_select_cb(ssl_ctx, select_h1_next_proto_cb, nullptr);
+
+#if OPENSSL_VERSION_NUMBER >= 0x10002000L
+    SSL_CTX_set_alpn_protos(
+        ssl_ctx, reinterpret_cast<const unsigned char *>(NGHTTP2_H1_1_ALPN),
+        str_size(NGHTTP2_H1_1_ALPN));
+#endif // OPENSSL_VERSION_NUMBER >= 0x10002000L
+  }
 
   return ssl_ctx;
 }
@@ -1270,15 +1302,7 @@ SSL_CTX *setup_server_ssl_context(std::vector<SSL_CTX *> &all_ssl_ctx,
   return ssl_ctx;
 }
 
-bool downstream_tls_enabled() {
-  auto no_tls = get_config()->conn.downstream.no_tls;
-
-  if (get_config()->client_mode) {
-    return !no_tls;
-  }
-
-  return get_config()->http2_bridge && !no_tls;
-}
+bool downstream_tls_enabled() { return !get_config()->conn.downstream.no_tls; }
 
 SSL_CTX *setup_client_ssl_context(
 #ifdef HAVE_NEVERBLEED

src/shrpx_worker.cc

@@ -73,6 +73,7 @@ Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx,
       dconn_pool_(get_config()->conn.downstream.addr_groups.size()),
       worker_stat_(get_config()->conn.downstream.addr_groups.size()),
       dgrps_(get_config()->conn.downstream.addr_groups.size()),
+      downstream_tls_session_cache_size_(0),
       loop_(loop),
       sv_ssl_ctx_(sv_ssl_ctx),
       cl_ssl_ctx_(cl_ssl_ctx),
@@ -116,6 +117,12 @@ Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx,
 Worker::~Worker() {
   ev_async_stop(loop_, &w_);
   ev_timer_stop(loop_, &mcpool_clear_timer_);
+
+  for (auto &p : downstream_tls_session_cache_) {
+    for (auto session : p.second) {
+      SSL_SESSION_free(session);
+    }
+  }
 }
 
 void Worker::schedule_clear_mcpool() {
@@ -300,4 +307,57 @@ mruby::MRubyContext *Worker::get_mruby_context() const {
 }
 #endif // HAVE_MRUBY
 
+void Worker::cache_downstream_tls_session(const DownstreamAddr *addr,
+                                          SSL_SESSION *session) {
+  auto &tlsconf = get_config()->tls;
+
+  auto max = tlsconf.downstream_session_cache_per_worker;
+  if (max == 0) {
+    return;
+  }
+
+  if (downstream_tls_session_cache_size_ >= max) {
+    // It is implementation dependent which item is returned from
+    // std::begin().  Probably, this depends on hash algorithm.  If it
+    // is random fashion, then we are mostly OK.
+    auto it = std::begin(downstream_tls_session_cache_);
+    assert(it != std::end(downstream_tls_session_cache_));
+    auto &v = (*it).second;
+    assert(!v.empty());
+    auto sess = v.front();
+    v.pop_front();
+    SSL_SESSION_free(sess);
+    if (v.empty()) {
+      downstream_tls_session_cache_.erase(it);
+    }
+  }
+
+  auto it = downstream_tls_session_cache_.find(addr);
+  if (it == std::end(downstream_tls_session_cache_)) {
+    std::tie(it, std::ignore) = downstream_tls_session_cache_.emplace(
+        addr, std::deque<SSL_SESSION *>());
+  }
+  (*it).second.push_back(session);
+  ++downstream_tls_session_cache_size_;
+}
+
+SSL_SESSION *Worker::reuse_downstream_tls_session(const DownstreamAddr *addr) {
+  auto it = downstream_tls_session_cache_.find(addr);
+  if (it == std::end(downstream_tls_session_cache_)) {
+    return nullptr;
+  }
+
+  auto &v = (*it).second;
+  assert(!v.empty());
+  auto session = v.back();
+  v.pop_back();
+  --downstream_tls_session_cache_size_;
+
+  if (v.empty()) {
+    downstream_tls_session_cache_.erase(it);
+  }
+
+  return session;
+}
+
 } // namespace shrpx

src/shrpx_worker.h

@@ -30,6 +30,8 @@
 #include <mutex>
 #include <vector>
 #include <random>
+#include <unordered_map>
+#include <deque>
 #include <thread>
 #ifndef NOTHREADS
 #include <future>
@@ -143,6 +145,17 @@ public:
   mruby::MRubyContext *get_mruby_context() const;
 #endif // HAVE_MRUBY
 
+  // Caches |session| which is associated to downstream address
+  // |addr|.  The caller is responsible to increment the reference
+  // count of |session|, since this function does not do so.
+  void cache_downstream_tls_session(const DownstreamAddr *addr,
+                                    SSL_SESSION *session);
+  // Returns cached session associated |addr|.  If non-nullptr value
+  // is returned, its cache entry was successfully removed from cache.
+  // If no cache entry is found associated to |addr|, nullptr will be
+  // returned.
+  SSL_SESSION *reuse_downstream_tls_session(const DownstreamAddr *addr);
+
 private:
 #ifndef NOTHREADS
   std::future<void> fut_;
@@ -156,6 +169,16 @@ private:
   DownstreamConnectionPool dconn_pool_;
   WorkerStat worker_stat_;
   std::vector<DownstreamGroup> dgrps_;
+
+  // Cache for SSL_SESSION for downstream connections.  SSL_SESSION is
+  // associated to downstream address.  One address has multiple
+  // SSL_SESSION objects.  New SSL_SESSION is appended to the deque.
+  // When doing eviction due to storage limitation, the SSL_SESSION
+  // which sits at the front of deque is removed.
+  std::unordered_map<const DownstreamAddr *, std::deque<SSL_SESSION *>>
+      downstream_tls_session_cache_;
+  size_t downstream_tls_session_cache_size_;
+
   std::unique_ptr<MemcachedDispatcher> session_cache_memcached_dispatcher_;
 #ifdef HAVE_MRUBY
   std::unique_ptr<mruby::MRubyContext> mruby_ctx_;