nghttpx: Cache TLS session inside DownstreamAddr object

src/shrpx_config.h

@@ -277,6 +277,14 @@ struct UpstreamAddr {
   int fd;
 };
 
+struct TLSSessionCache {
+  // ASN1 representation of SSL_SESSION object.  See
+  // i2d_SSL_SESSION(3SSL).
+  std::vector<uint8_t> session_data;
+  // The last time stamp when this cache entry is created or updated.
+  ev_tstamp last_updated;
+};
+
 struct DownstreamAddr {
   Address addr;
   // backend address.  If |host_unix| is true, this is UNIX domain
@@ -284,6 +292,8 @@ struct DownstreamAddr {
   ImmutableString host;
   ImmutableString hostport;
   ConnectBlocker *connect_blocker;
+  // Client side TLS session cache
+  TLSSessionCache tls_session_cache;
   // backend port.  0 if |host_unix| is true.
   uint16_t port;
   // true if |host| contains UNIX domain socket path.

src/shrpx_http_downstream_connection.cc

@@ -129,15 +129,7 @@ HttpDownstreamConnection::HttpDownstreamConnection(
       response_htp_{0},
       group_(group) {}
 
-HttpDownstreamConnection::~HttpDownstreamConnection() {
-  if (conn_.tls.ssl && conn_.tls.initial_handshake_done) {
-    auto session = SSL_get0_session(conn_.tls.ssl);
-    if (session) {
-      worker_->cache_client_tls_session(&addr_->addr, session,
-                                        ev_now(conn_.loop));
-    }
-  }
-}
+HttpDownstreamConnection::~HttpDownstreamConnection() {}
 
 int HttpDownstreamConnection::attach_downstream(Downstream *downstream) {
   if (LOG_ENABLED(INFO)) {
@@ -241,7 +233,7 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) {
           SSL_set_tlsext_host_name(conn_.tls.ssl, sni_name.c_str());
         }
 
-        auto session = worker_->reuse_client_tls_session(&addr_->addr);
+        auto session = ssl::reuse_tls_session(addr_);
         if (session) {
           SSL_set_session(conn_.tls.ssl, session);
           SSL_SESSION_free(session);
@@ -891,6 +883,13 @@ int HttpDownstreamConnection::tls_handshake() {
     return -1;
   }
 
+  if (!SSL_session_reused(conn_.tls.ssl)) {
+    auto session = SSL_get0_session(conn_.tls.ssl);
+    if (session) {
+      ssl::try_cache_tls_session(addr_, session, ev_now(conn_.loop));
+    }
+  }
+
   do_read_ = &HttpDownstreamConnection::read_tls;
   do_write_ = &HttpDownstreamConnection::write_tls;
 

src/shrpx_http_downstream_connection.h

@@ -82,7 +82,7 @@ private:
   // nullptr if TLS is not used.
   SSL_CTX *ssl_ctx_;
   // Address of remote endpoint
-  const DownstreamAddr *addr_;
+  DownstreamAddr *addr_;
   IOControl ioctrl_;
   http_parser response_htp_;
   size_t group_;

src/shrpx_ssl.cc

@@ -36,6 +36,7 @@
 
 #include <vector>
 #include <string>
+#include <iomanip>
 
 #include <openssl/crypto.h>
 #include <openssl/x509.h>
@@ -1335,6 +1336,50 @@ CertLookupTree *create_cert_lookup_tree() {
   return new ssl::CertLookupTree();
 }
 
+namespace {
+std::vector<uint8_t> serialize_ssl_session(SSL_SESSION *session) {
+  auto len = i2d_SSL_SESSION(session, nullptr);
+  auto buf = std::vector<uint8_t>(len);
+  auto p = buf.data();
+  i2d_SSL_SESSION(session, &p);
+
+  return buf;
+}
+} // namespace
+
+void try_cache_tls_session(DownstreamAddr *addr, SSL_SESSION *session,
+                           ev_tstamp t) {
+  auto &cache = addr->tls_session_cache;
+
+  if (cache.last_updated + 1_min > t) {
+    if (LOG_ENABLED(INFO)) {
+      LOG(INFO) << "Cache for addr=" << util::to_numeric_addr(&addr->addr)
+                << " is still host.  Not updating.";
+    }
+    return;
+  }
+
+  if (LOG_ENABLED(INFO)) {
+    LOG(INFO) << "Update cache entry for SSL_SESSION=" << session
+              << ", addr=" << util::to_numeric_addr(&addr->addr)
+              << ", timestamp=" << std::fixed << std::setprecision(6) << t;
+  }
+
+  cache.session_data = serialize_ssl_session(session);
+  cache.last_updated = t;
+}
+
+SSL_SESSION *reuse_tls_session(const DownstreamAddr *addr) {
+  auto &cache = addr->tls_session_cache;
+
+  if (cache.session_data.empty()) {
+    return nullptr;
+  }
+
+  auto p = cache.session_data.data();
+  return d2i_SSL_SESSION(nullptr, &p, cache.session_data.size());
+}
+
 } // namespace ssl
 
 } // namespace shrpx

src/shrpx_ssl.h

@@ -217,6 +217,17 @@ bool downstream_tls_enabled();
 bool tls_hostname_match(const char *pattern, size_t plen, const char *hostname,
                         size_t hlen);
 
+// Caches |session| which is associated to remote address |addr|.
+// |session| is serialized into ASN1 representation, and stored.  |t|
+// is used as a time stamp.  Depending on the existing cache's time
+// stamp, |session| might not be cached.
+void try_cache_tls_session(DownstreamAddr *addr, SSL_SESSION *session,
+                           ev_tstamp t);
+
+// Returns cached session associated |addr|.  If no cache entry is
+// found associated to |addr|, nullptr will be returned.
+SSL_SESSION *reuse_tls_session(const DownstreamAddr *addr);
+
 } // namespace ssl
 
 } // namespace shrpx

src/shrpx_worker.cc

@@ -29,7 +29,6 @@
 #endif // HAVE_UNISTD_H
 
 #include <memory>
-#include <iomanip>
 
 #include "shrpx_ssl.h"
 #include "shrpx_log.h"
@@ -312,61 +311,6 @@ mruby::MRubyContext *Worker::get_mruby_context() const {
 }
 #endif // HAVE_MRUBY
 
-namespace {
-std::vector<uint8_t> serialize_ssl_session(SSL_SESSION *session) {
-  auto len = i2d_SSL_SESSION(session, nullptr);
-  auto buf = std::vector<uint8_t>(len);
-  auto p = buf.data();
-  i2d_SSL_SESSION(session, &p);
-
-  return buf;
-}
-} // namespace
-
-void Worker::cache_client_tls_session(const Address *addr, SSL_SESSION *session,
-                                      ev_tstamp t) {
-  auto it = client_tls_session_cache_.find(addr);
-  if (it == std::end(client_tls_session_cache_)) {
-    if (LOG_ENABLED(INFO)) {
-      LOG(INFO) << "Create cache entry for SSL_SESSION=" << session
-                << ", addr=" << util::to_numeric_addr(addr) << "(" << addr
-                << "), timestamp=" << std::fixed << std::setprecision(6) << t;
-    }
-    client_tls_session_cache_.emplace(
-        addr, SessionCacheEntry{serialize_ssl_session(session), t});
-    return;
-  }
-
-  auto &ent = (*it).second;
-  if (ent.last_updated + 1_min > t) {
-    if (LOG_ENABLED(INFO)) {
-      LOG(INFO) << "Cache for addr=" << util::to_numeric_addr(addr) << "("
-                << addr << ") is still host.  Not updating.";
-    }
-    return;
-  }
-
-  if (LOG_ENABLED(INFO)) {
-    LOG(INFO) << "Update cache entry for SSL_SESSION=" << session
-              << ", addr=" << util::to_numeric_addr(addr) << "(" << addr
-              << "), timestamp=" << std::fixed << std::setprecision(6) << t;
-  }
-
-  ent.session_data = serialize_ssl_session(session);
-  ent.last_updated = t;
-}
-
-SSL_SESSION *Worker::reuse_client_tls_session(const Address *addr) {
-  auto it = client_tls_session_cache_.find(addr);
-  if (it == std::end(client_tls_session_cache_)) {
-    return nullptr;
-  }
-
-  const auto &ent = (*it).second;
-  auto p = ent.session_data.data();
-  return d2i_SSL_SESSION(nullptr, &p, ent.session_data.size());
-}
-
 std::vector<DownstreamAddrGroup> &Worker::get_downstream_addr_groups() {
   return downstream_addr_groups_;
 }

src/shrpx_worker.h

@@ -101,14 +101,6 @@ struct WorkerEvent {
   std::shared_ptr<TicketKeys> ticket_keys;
 };
 
-struct SessionCacheEntry {
-  // ASN1 representation of SSL_SESSION object.  See
-  // i2d_SSL_SESSION(3SSL).
-  std::vector<uint8_t> session_data;
-  // The last time stamp when this cache entry is created or updated.
-  ev_tstamp last_updated;
-};
-
 class Worker {
 public:
   Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx,
@@ -153,16 +145,6 @@ public:
   mruby::MRubyContext *get_mruby_context() const;
 #endif // HAVE_MRUBY
 
-  // Caches |session| which is associated to remote address |addr|.
-  // |session| is serialized into ASN1 representation, and stored.
-  // |t| is used as a time stamp.  Depending on the existing cache's
-  // time stamp, |session| might not be cached.
-  void cache_client_tls_session(const Address *addr, SSL_SESSION *session,
-                                ev_tstamp t);
-  // Returns cached session associated |addr|.  If no cache entry is
-  // found associated to |addr|, nullptr will be returned.
-  SSL_SESSION *reuse_client_tls_session(const Address *addr);
-
   std::vector<DownstreamAddrGroup> &get_downstream_addr_groups();
 
   ConnectBlocker *get_connect_blocker() const;
@@ -181,11 +163,6 @@ private:
   WorkerStat worker_stat_;
   std::vector<DownstreamGroup> dgrps_;
 
-  // Client side SSL_SESSION cache.  SSL_SESSION is associated to
-  // remote address.
-  std::unordered_map<const Address *, SessionCacheEntry>
-      client_tls_session_cache_;
-
   std::unique_ptr<MemcachedDispatcher> session_cache_memcached_dispatcher_;
 #ifdef HAVE_MRUBY
   std::unique_ptr<mruby::MRubyContext> mruby_ctx_;