nghttpx: Add --tls-ctx-per-worker option

When same SSL_CTX is used by multiple thread simultaneously we have to
setup some number of mutex locks for it.  We could not check how this
locking affects scalability since we have 4 cores at best in our
development machine.  Good side of sharing SSL_CTX across threads is
we can share session ID pool.

If --tls-ctx-per-worker is enabled, SSL_CTX is created per thread
basis and we can eliminate mutex locks.  The downside is session ID is
no longer shared, which means if session ID generated by one thread
cannot be acceptable by another thread.  But we have now session
ticket enabled and its keys are shared by all threads.

src/shrpx.cc

@@ -485,23 +485,8 @@ void renew_ticket_key_cb(struct ev_loop *loop, ev_timer *w, int revents) {
 namespace {
 int event_loop() {
   auto loop = EV_DEFAULT;
-  SSL_CTX *sv_ssl_ctx, *cl_ssl_ctx;
 
-  if (get_config()->client_mode) {
-    sv_ssl_ctx = nullptr;
-    cl_ssl_ctx = get_config()->downstream_no_tls
-                     ? nullptr
-                     : ssl::create_ssl_client_context();
-  } else {
-    sv_ssl_ctx =
-        get_config()->upstream_no_tls ? nullptr : get_config()->default_ssl_ctx;
-    cl_ssl_ctx = get_config()->http2_bridge && !get_config()->downstream_no_tls
-                     ? ssl::create_ssl_client_context()
-                     : nullptr;
-  }
-
-  auto conn_handler =
-      util::make_unique<ConnectionHandler>(loop, sv_ssl_ctx, cl_ssl_ctx);
+  auto conn_handler = util::make_unique<ConnectionHandler>(loop);
   if (get_config()->daemon) {
     if (daemon(0, 0) == -1) {
       auto error = errno;
@@ -533,7 +518,8 @@ int event_loop() {
   drop_privileges();
 
   ev_timer renew_ticket_key_timer;
-  if (sv_ssl_ctx && get_config()->auto_tls_ticket_key) {
+  if (!get_config()->client_mode && !get_config()->upstream_no_tls &&
+      get_config()->auto_tls_ticket_key) {
     // Renew ticket key every 12hrs
     ev_timer_init(&renew_ticket_key_timer, renew_ticket_key_cb, 0., 12 * 3600.);
     renew_ticket_key_timer.data = conn_handler.get();
@@ -556,6 +542,10 @@ int event_loop() {
   }
 #endif // !NOTHREADS
 
+  if (!get_config()->tls_ctx_per_worker) {
+    conn_handler->create_ssl_context();
+  }
+
   if (get_config()->num_worker > 1) {
     conn_handler->create_worker_thread(get_config()->num_worker);
   } else if (get_config()->downstream_proto == PROTO_HTTP2) {
@@ -719,7 +709,6 @@ void fill_default_config() {
   mod_config()->pid = getpid();
   mod_config()->backend_ipv4 = false;
   mod_config()->backend_ipv6 = false;
-  mod_config()->cert_tree = nullptr;
   mod_config()->downstream_http_proxy_userinfo = nullptr;
   mod_config()->downstream_http_proxy_host = nullptr;
   mod_config()->downstream_http_proxy_port = 0;
@@ -755,6 +744,7 @@ void fill_default_config() {
   mod_config()->downstream_connections_per_frontend = 0;
   mod_config()->listener_disable_timeout = 0.;
   mod_config()->auto_tls_ticket_key = true;
+  mod_config()->tls_ctx_per_worker = false;
 }
 } // namespace
 
@@ -1016,6 +1006,16 @@ SSL/TLS:
                      while opening or reading a file, key is generated
                      automatically and renewed every 12hrs.  At most 2
                      keys are stored in memory.
+  --tls-ctx-per-worker
+                     Create OpenSSL's  SSL_CTX per worker, so  that no
+                     internal locking  is required.  This  may improve
+                     scalability  with  multi threaded  configuration.
+                     If  this  option is  enabled,  session  ID is  no
+                     longer  shared  accross  SSL_CTX  objects,  which
+                     means session  ID generated by one  worker is not
+                     acceptable by another worker.  On the other hand,
+                     session ticket  key is  shared across  all worker
+                     threads.
 
 HTTP/2 and SPDY:
   -c, --http2-max-concurrent-streams=<NUM>
@@ -1288,6 +1288,7 @@ int main(int argc, char **argv) {
          67},
         {"tls-ticket-key-file", required_argument, &flag, 68},
         {"rlimit-nofile", required_argument, &flag, 69},
+        {"tls-ctx-per-worker", no_argument, &flag, 70},
         {nullptr, 0, nullptr, 0}};
 
     int option_index = 0;
@@ -1605,6 +1606,10 @@ int main(int argc, char **argv) {
         // --rlimit-nofile
         cmdcfgs.emplace_back(SHRPX_OPT_RLIMIT_NOFILE, optarg);
         break;
+      case 70:
+        // --tls-ctx-per-worker
+        cmdcfgs.emplace_back(SHRPX_OPT_TLS_CTX_PER_WORKER, "yes");
+        break;
       default:
         break;
       }
@@ -1620,9 +1625,6 @@ int main(int argc, char **argv) {
   OpenSSL_add_all_algorithms();
   SSL_load_error_strings();
   SSL_library_init();
-#ifndef NOTHREADS
-  nghttp2::ssl::LibsslGlobalLock lock;
-#endif // NOTHREADS
 
   if (conf_exists(get_config()->conf_path.get())) {
     if (load_config(get_config()->conf_path.get()) == -1) {
@@ -1648,6 +1650,13 @@ int main(int argc, char **argv) {
     }
   }
 
+#ifndef NOTHREADS
+  std::unique_ptr<nghttp2::ssl::LibsslGlobalLock> lock;
+  if (!get_config()->tls_ctx_per_worker) {
+    lock = util::make_unique<nghttp2::ssl::LibsslGlobalLock>();
+  }
+#endif // NOTHREADS
+
   if (get_config()->accesslog_syslog || get_config()->errorlog_syslog) {
     openlog("nghttpx", LOG_NDELAY | LOG_NOWAIT | LOG_PID,
             get_config()->syslog_facility);
@@ -1730,33 +1739,6 @@ int main(int argc, char **argv) {
 
   mod_config()->alpn_prefs = ssl::set_alpn_prefs(get_config()->npn_list);
 
-  if (!get_config()->subcerts.empty()) {
-    mod_config()->cert_tree = ssl::cert_lookup_tree_new();
-  }
-
-  for (auto &keycert : get_config()->subcerts) {
-    auto ssl_ctx =
-        ssl::create_ssl_context(keycert.first.c_str(), keycert.second.c_str());
-    if (ssl::cert_lookup_tree_add_cert_from_file(
-            get_config()->cert_tree, ssl_ctx, keycert.second.c_str()) == -1) {
-      LOG(FATAL) << "Failed to add sub certificate.";
-      exit(EXIT_FAILURE);
-    }
-  }
-
-  if (get_config()->cert_file && get_config()->private_key_file) {
-    mod_config()->default_ssl_ctx = ssl::create_ssl_context(
-        get_config()->private_key_file.get(), get_config()->cert_file.get());
-    if (get_config()->cert_tree) {
-      if (ssl::cert_lookup_tree_add_cert_from_file(
-              get_config()->cert_tree, get_config()->default_ssl_ctx,
-              get_config()->cert_file.get()) == -1) {
-        LOG(FATAL) << "Failed to add server certificate to the lookup tree.";
-        exit(EXIT_FAILURE);
-      }
-    }
-  }
-
   if (!get_config()->tls_ticket_key_files.empty()) {
     auto ticket_keys =
         read_tls_ticket_key_file(get_config()->tls_ticket_key_files);

src/shrpx_config.cc

@@ -140,6 +140,7 @@ const char SHRPX_OPT_BACKEND_HTTP1_CONNECTIONS_PER_FRONTEND[] =
 const char SHRPX_OPT_LISTENER_DISABLE_TIMEOUT[] = "listener-disable-timeout";
 const char SHRPX_OPT_TLS_TICKET_KEY_FILE[] = "tls-ticket-key-file";
 const char SHRPX_OPT_RLIMIT_NOFILE[] = "rlimit-nofile";
+const char SHRPX_OPT_TLS_CTX_PER_WORKER[] = "tls-ctx-per-worker";
 
 namespace {
 Config *config = nullptr;
@@ -1128,6 +1129,12 @@ int parse_config(const char *opt, const char *optarg) {
     return 0;
   }
 
+  if (util::strieq(opt, SHRPX_OPT_TLS_CTX_PER_WORKER)) {
+    mod_config()->tls_ctx_per_worker = util::strieq(optarg, "yes");
+
+    return 0;
+  }
+
   if (util::strieq(opt, "conf")) {
     LOG(WARN) << "conf: ignored";
 

src/shrpx_config.h

@@ -128,6 +128,7 @@ extern const char SHRPX_OPT_BACKEND_HTTP1_CONNECTIONS_PER_FRONTEND[];
 extern const char SHRPX_OPT_LISTENER_DISABLE_TIMEOUT[];
 extern const char SHRPX_OPT_TLS_TICKET_KEY_FILE[];
 extern const char SHRPX_OPT_RLIMIT_NOFILE[];
+extern const char SHRPX_OPT_TLS_CTX_PER_WORKER[];
 
 union sockaddr_union {
   sockaddr sa;
@@ -199,8 +200,6 @@ struct Config {
   std::unique_ptr<char[]> private_key_passwd;
   std::unique_ptr<char[]> cert_file;
   std::unique_ptr<char[]> dh_param_file;
-  SSL_CTX *default_ssl_ctx;
-  ssl::CertLookupTree *cert_tree;
   const char *server_name;
   std::unique_ptr<char[]> backend_tls_sni_name;
   std::unique_ptr<char[]> pid_file;
@@ -296,6 +295,7 @@ struct Config {
   bool upstream_frame_debug;
   bool no_location_rewrite;
   bool auto_tls_ticket_key;
+  bool tls_ctx_per_worker;
 };
 
 const Config *get_config();

src/shrpx_connection_handler.cc

@@ -58,9 +58,8 @@ void acceptor_disable_cb(struct ev_loop *loop, ev_timer *w, int revent) {
 }
 } // namespace
 
-ConnectionHandler::ConnectionHandler(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx,
-                                     SSL_CTX *cl_ssl_ctx)
-    : loop_(loop), sv_ssl_ctx_(sv_ssl_ctx), cl_ssl_ctx_(cl_ssl_ctx),
+ConnectionHandler::ConnectionHandler(struct ev_loop *loop)
+    : loop_(loop), sv_ssl_ctx_(nullptr), cl_ssl_ctx_(nullptr),
       // rate_limit_group_(bufferevent_rate_limit_group_new(
       //     evbase, get_config()->worker_rate_limit_cfg)),
       worker_stat_(util::make_unique<WorkerStat>()),
@@ -74,6 +73,11 @@ ConnectionHandler::~ConnectionHandler() {
   ev_timer_stop(loop_, &disable_acceptor_timer_);
 }
 
+void ConnectionHandler::create_ssl_context() {
+  sv_ssl_ctx_ = ssl::setup_server_ssl_context();
+  cl_ssl_ctx_ = ssl::setup_client_ssl_context();
+}
+
 void ConnectionHandler::worker_reopen_log_files() {
   WorkerEvent wev;
 
@@ -104,6 +108,7 @@ void ConnectionHandler::create_worker_thread(size_t num) {
 
   for (size_t i = 0; i < num; ++i) {
     workers_.push_back(util::make_unique<Worker>(sv_ssl_ctx_, cl_ssl_ctx_,
+                                                 worker_config->cert_tree,
                                                  worker_config->ticket_keys));
 
     if (LOG_ENABLED(INFO)) {

src/shrpx_connection_handler.h

@@ -51,10 +51,10 @@ struct TicketKeys;
 // TODO should be renamed as ConnectionHandler
 class ConnectionHandler {
 public:
-  ConnectionHandler(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx,
-                    SSL_CTX *cl_ssl_ctx);
+  ConnectionHandler(struct ev_loop *loop);
   ~ConnectionHandler();
   int handle_connection(int fd, sockaddr *addr, int addrlen);
+  void create_ssl_context();
   void create_worker_thread(size_t num);
   void worker_reopen_log_files();
   void worker_renew_ticket_keys(const std::shared_ptr<TicketKeys> &ticket_keys);

src/shrpx_ssl.cc

@@ -129,11 +129,12 @@ int ssl_pem_passwd_cb(char *buf, int size, int rwflag, void *user_data) {
 
 namespace {
 int servername_callback(SSL *ssl, int *al, void *arg) {
-  if (get_config()->cert_tree) {
+  auto cert_tree = worker_config->cert_tree;
+  if (cert_tree) {
     const char *hostname = SSL_get_servername(ssl, TLSEXT_NAMETYPE_host_name);
     if (hostname) {
-      auto ssl_ctx = cert_lookup_tree_lookup(get_config()->cert_tree, hostname,
-                                             strlen(hostname));
+      auto ssl_ctx =
+          cert_lookup_tree_lookup(cert_tree, hostname, strlen(hostname));
       if (ssl_ctx) {
         SSL_set_SSL_CTX(ssl, ssl_ctx);
       }
@@ -943,6 +944,50 @@ bool check_http2_requirement(SSL *ssl) {
   return true;
 }
 
+SSL_CTX *setup_server_ssl_context() {
+  if (get_config()->upstream_no_tls) {
+    return nullptr;
+  }
+
+  auto ssl_ctx = ssl::create_ssl_context(get_config()->private_key_file.get(),
+                                         get_config()->cert_file.get());
+
+  auto cert_tree =
+      get_config()->subcerts.empty() ? nullptr : cert_lookup_tree_new();
+  worker_config->cert_tree = cert_tree;
+
+  for (auto &keycert : get_config()->subcerts) {
+    auto ssl_ctx =
+        ssl::create_ssl_context(keycert.first.c_str(), keycert.second.c_str());
+    if (ssl::cert_lookup_tree_add_cert_from_file(
+            cert_tree, ssl_ctx, keycert.second.c_str()) == -1) {
+      LOG(FATAL) << "Failed to add sub certificate.";
+      DIE();
+    }
+  }
+
+  if (cert_tree) {
+    if (ssl::cert_lookup_tree_add_cert_from_file(
+            cert_tree, ssl_ctx, get_config()->cert_file.get()) == -1) {
+      LOG(FATAL) << "Failed to add default certificate.";
+      DIE();
+    }
+  }
+
+  return ssl_ctx;
+}
+
+SSL_CTX *setup_client_ssl_context() {
+  if (get_config()->client_mode) {
+    return get_config()->downstream_no_tls ? nullptr
+                                           : ssl::create_ssl_client_context();
+  }
+
+  return get_config()->http2_bridge && !get_config()->downstream_no_tls
+             ? ssl::create_ssl_client_context()
+             : nullptr;
+}
+
 } // namespace ssl
 
 } // namespace shrpx

src/shrpx_ssl.h

@@ -141,6 +141,17 @@ long int create_tls_proto_mask(const std::vector<char *> &tls_proto_list);
 
 std::vector<unsigned char> set_alpn_prefs(const std::vector<char *> &protos);
 
+// Setups server side SSL_CTX.  This function inspects get_config()
+// and if upstream_no_tls is true, returns nullptr.  Otherwise
+// construct default SSL_CTX.  If subcerts are not empty, create
+// SSL_CTX for them.  All created SSL_CTX are added to CertLookupTree.
+SSL_CTX *setup_server_ssl_context();
+
+// Setups client side SSL_CTX.  This function inspects get_config()
+// and if downstream_no_tls is true, returns nullptr.  Otherwise, only
+// construct SSL_CTX if either client_mode or http2_bridge is true.
+SSL_CTX *setup_client_ssl_context();
+
 } // namespace ssl
 
 } // namespace shrpx

src/shrpx_worker.cc

@@ -48,6 +48,7 @@ void eventcb(struct ev_loop *loop, ev_async *w, int revents) {
 } // namespace
 
 Worker::Worker(SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx,
+               ssl::CertLookupTree *cert_tree,
                const std::shared_ptr<TicketKeys> &ticket_keys)
     : loop_(ev_loop_new(0)), sv_ssl_ctx_(sv_ssl_ctx), cl_ssl_ctx_(cl_ssl_ctx),
       worker_stat_(util::make_unique<WorkerStat>()) {
@@ -55,14 +56,21 @@ Worker::Worker(SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx,
   w_.data = this;
   ev_async_start(loop_, &w_);
 
-  if (get_config()->downstream_proto == PROTO_HTTP2) {
-    http2session_ = util::make_unique<Http2Session>(loop_, cl_ssl_ctx_);
-  } else {
-    http1_connect_blocker_ = util::make_unique<ConnectBlocker>(loop_);
-  }
-
 #ifndef NOTHREADS
-  fut_ = std::async(std::launch::async, [this, &ticket_keys] {
+  fut_ = std::async(std::launch::async, [this, cert_tree, &ticket_keys] {
+    if (get_config()->tls_ctx_per_worker) {
+      sv_ssl_ctx_ = ssl::setup_server_ssl_context();
+      cl_ssl_ctx_ = ssl::setup_client_ssl_context();
+    } else {
+      worker_config->cert_tree = cert_tree;
+    }
+
+    if (get_config()->downstream_proto == PROTO_HTTP2) {
+      http2session_ = util::make_unique<Http2Session>(loop_, cl_ssl_ctx_);
+    } else {
+      http1_connect_blocker_ = util::make_unique<ConnectBlocker>(loop_);
+    }
+
     worker_config->ticket_keys = ticket_keys;
     (void)reopen_log_files();
     ev_run(loop_);

src/shrpx_worker.h

@@ -47,6 +47,10 @@ namespace shrpx {
 class Http2Session;
 class ConnectBlocker;
 
+namespace ssl {
+struct CertLookupTree;
+} // namespace ssl
+
 struct WorkerStat {
   WorkerStat() : num_connections(0), next_downstream(0) {}
 
@@ -77,6 +81,7 @@ struct WorkerEvent {
 class Worker {
 public:
   Worker(SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx,
+         ssl::CertLookupTree *cert_tree,
          const std::shared_ptr<TicketKeys> &ticket_keys);
   ~Worker();
   void wait();

src/shrpx_worker_config.cc

@@ -27,8 +27,8 @@
 namespace shrpx {
 
 WorkerConfig::WorkerConfig()
-    : accesslog_fd(-1), errorlog_fd(-1), errorlog_tty(false),
-      graceful_shutdown(false) {}
+    : cert_tree(nullptr), accesslog_fd(-1), errorlog_fd(-1),
+      errorlog_tty(false), graceful_shutdown(false) {}
 
 #ifndef NOTHREADS
 thread_local

src/shrpx_worker_config.h

@@ -29,10 +29,15 @@
 
 namespace shrpx {
 
+namespace ssl {
+struct CertLookupTree;
+} // namespace ssl
+
 struct TicketKeys;
 
 struct WorkerConfig {
   std::shared_ptr<TicketKeys> ticket_keys;
+  ssl::CertLookupTree *cert_tree;
   int accesslog_fd;
   int errorlog_fd;
   // true if errorlog_fd is referring to a terminal.