nghttpx: Single SSL_SESSION cache entry for each address

src/shrpx_http_downstream_connection.cc

@@ -131,9 +131,10 @@ HttpDownstreamConnection::HttpDownstreamConnection(
 
 HttpDownstreamConnection::~HttpDownstreamConnection() {
   if (conn_.tls.ssl) {
-    auto session = SSL_get1_session(conn_.tls.ssl);
+    auto session = SSL_get0_session(conn_.tls.ssl);
     if (session) {
-      worker_->cache_downstream_tls_session(&addr_->addr, session);
+      worker_->cache_downstream_tls_session(&addr_->addr, session,
+                                            ev_now(conn_.loop));
     }
   }
 }

src/shrpx_worker.cc

@@ -29,6 +29,7 @@
 #endif // HAVE_UNISTD_H
 
 #include <memory>
+#include <iomanip>
 
 #include "shrpx_ssl.h"
 #include "shrpx_log.h"
@@ -73,7 +74,6 @@ Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx,
       dconn_pool_(get_config()->conn.downstream.addr_groups.size()),
       worker_stat_(get_config()->conn.downstream.addr_groups.size()),
       dgrps_(get_config()->conn.downstream.addr_groups.size()),
-      downstream_tls_session_cache_size_(0),
       loop_(loop),
       sv_ssl_ctx_(sv_ssl_ctx),
       cl_ssl_ctx_(cl_ssl_ctx),
@@ -117,12 +117,6 @@ Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx,
 Worker::~Worker() {
   ev_async_stop(loop_, &w_);
   ev_timer_stop(loop_, &mcpool_clear_timer_);
-
-  for (auto &p : downstream_tls_session_cache_) {
-    for (auto session : p.second) {
-      SSL_SESSION_free(session);
-    }
-  }
 }
 
 void Worker::schedule_clear_mcpool() {
@@ -307,38 +301,51 @@ mruby::MRubyContext *Worker::get_mruby_context() const {
 }
 #endif // HAVE_MRUBY
 
-void Worker::cache_downstream_tls_session(const Address *addr,
-                                          SSL_SESSION *session) {
-  auto &tlsconf = get_config()->tls;
+namespace {
+std::vector<uint8_t> serialize_ssl_session(SSL_SESSION *session) {
+  auto len = i2d_SSL_SESSION(session, nullptr);
+  auto buf = std::vector<uint8_t>(len);
+  auto p = buf.data();
+  i2d_SSL_SESSION(session, &p);
 
-  auto max = tlsconf.downstream_session_cache_per_worker;
-  if (max == 0) {
+  return buf;
+}
+} // namespace
+
+void Worker::cache_downstream_tls_session(const Address *addr,
+                                          SSL_SESSION *session, ev_tstamp t) {
+  auto it = downstream_tls_session_cache_.find(addr);
+  if (it == std::end(downstream_tls_session_cache_)) {
+    if (LOG_ENABLED(INFO)) {
+      LOG(INFO) << "Create cache entry for SSL_SESSION=" << session
+                << ", addr=" << util::numeric_hostport(&addr->su.sa, addr->len)
+                << "(" << addr << "), timestamp=" << std::fixed
+                << std::setprecision(6) << t;
+    }
+    downstream_tls_session_cache_.emplace(
+        addr, SessionCacheEntry{serialize_ssl_session(session), t});
     return;
   }
 
-  if (downstream_tls_session_cache_size_ >= max) {
-    // It is implementation dependent which item is returned from
-    // std::begin().  Probably, this depends on hash algorithm.  If it
-    // is random fashion, then we are mostly OK.
-    auto it = std::begin(downstream_tls_session_cache_);
-    assert(it != std::end(downstream_tls_session_cache_));
-    auto &v = (*it).second;
-    assert(!v.empty());
-    auto sess = v.front();
-    v.pop_front();
-    SSL_SESSION_free(sess);
-    if (v.empty()) {
-      downstream_tls_session_cache_.erase(it);
+  auto &ent = (*it).second;
+  if (ent.last_updated + 1_min > t) {
+    if (LOG_ENABLED(INFO)) {
+      LOG(INFO) << "Cache for addr="
+                << util::numeric_hostport(&addr->su.sa, addr->len) << "("
+                << addr << ") is still host.  Not updating.";
     }
+    return;
   }
 
-  auto it = downstream_tls_session_cache_.find(addr);
-  if (it == std::end(downstream_tls_session_cache_)) {
-    std::tie(it, std::ignore) = downstream_tls_session_cache_.emplace(
-        addr, std::deque<SSL_SESSION *>());
+  if (LOG_ENABLED(INFO)) {
+    LOG(INFO) << "Update cache entry for SSL_SESSION=" << session
+              << ", addr=" << util::numeric_hostport(&addr->su.sa, addr->len)
+              << "(" << addr << "), timestamp=" << std::fixed
+              << std::setprecision(6) << t;
   }
-  (*it).second.push_back(session);
-  ++downstream_tls_session_cache_size_;
+
+  ent.session_data = serialize_ssl_session(session);
+  ent.last_updated = t;
 }
 
 SSL_SESSION *Worker::reuse_downstream_tls_session(const Address *addr) {
@@ -347,17 +354,9 @@ SSL_SESSION *Worker::reuse_downstream_tls_session(const Address *addr) {
     return nullptr;
   }
 
-  auto &v = (*it).second;
-  assert(!v.empty());
-  auto session = v.back();
-  v.pop_back();
-  --downstream_tls_session_cache_size_;
-
-  if (v.empty()) {
-    downstream_tls_session_cache_.erase(it);
-  }
-
-  return session;
+  const auto &ent = (*it).second;
+  auto p = ent.session_data.data();
+  return d2i_SSL_SESSION(nullptr, &p, ent.session_data.size());
 }
 
 } // namespace shrpx

src/shrpx_worker.h

@@ -101,6 +101,14 @@ struct WorkerEvent {
   std::shared_ptr<TicketKeys> ticket_keys;
 };
 
+struct SessionCacheEntry {
+  // ASN1 representation of SSL_SESSION object.  See
+  // i2d_SSL_SESSION(3SSL).
+  std::vector<uint8_t> session_data;
+  // The last time stamp when this cache entry is created or updated.
+  ev_tstamp last_updated;
+};
+
 class Worker {
 public:
   Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx,
@@ -146,13 +154,13 @@ public:
 #endif // HAVE_MRUBY
 
   // Caches |session| which is associated to remote address |addr|.
-  // The caller is responsible to increment the reference count of
-  // |session|, since this function does not do so.
-  void cache_downstream_tls_session(const Address *addr, SSL_SESSION *session);
-  // Returns cached session associated |addr|.  If non-nullptr value
-  // is returned, its cache entry was successfully removed from cache.
-  // If no cache entry is found associated to |addr|, nullptr will be
-  // returned.
+  // |session| is serialized into ASN1 representation, and stored.
+  // |t| is used as a time stamp.  Depending on the existing cache's
+  // time stamp, |session| might not be cached.
+  void cache_downstream_tls_session(const Address *addr, SSL_SESSION *session,
+                                    ev_tstamp t);
+  // Returns cached session associated |addr|.  If no cache entry is
+  // found associated to |addr|, nullptr will be returned.
   SSL_SESSION *reuse_downstream_tls_session(const Address *addr);
 
 private:
@@ -170,13 +178,9 @@ private:
   std::vector<DownstreamGroup> dgrps_;
 
   // Client side SSL_SESSION cache.  SSL_SESSION is associated to
-  // remote address.  One address has multiple SSL_SESSION objects.
-  // New SSL_SESSION is appended to the deque.  When doing eviction
-  // due to storage limitation, the SSL_SESSION which sits at the
-  // front of deque is removed.
-  std::unordered_map<const Address *, std::deque<SSL_SESSION *>>
+  // remote address.
+  std::unordered_map<const Address *, SessionCacheEntry>
       downstream_tls_session_cache_;
-  size_t downstream_tls_session_cache_size_;
 
   std::unique_ptr<MemcachedDispatcher> session_cache_memcached_dispatcher_;
 #ifdef HAVE_MRUBY

src/util.cc

@@ -651,6 +651,41 @@ std::string numeric_name(const struct sockaddr *sa, socklen_t salen) {
   return host.data();
 }
 
+std::string numeric_hostport(const struct sockaddr *sa, socklen_t salen) {
+  if (sa->sa_family == AF_UNIX) {
+    return "localhost";
+  }
+
+  std::array<char, NI_MAXHOST> host;
+  std::array<char, NI_MAXSERV> serv;
+  auto rv = getnameinfo(sa, salen, host.data(), host.size(), serv.data(),
+                        serv.size(), NI_NUMERICHOST | NI_NUMERICSERV);
+  if (rv != 0) {
+    return "unknown";
+  }
+
+  auto hostlen = strlen(host.data());
+  auto servlen = strlen(serv.data());
+
+  std::string s;
+  char *p;
+  if (sa->sa_family == AF_INET6) {
+    s.resize(hostlen + servlen + 2 + 1);
+    p = &s[0];
+    *p++ = '[';
+    p = std::copy_n(host.data(), hostlen, p);
+    *p++ = ']';
+  } else {
+    s.resize(hostlen + servlen + 1);
+    p = &s[0];
+    p = std::copy_n(host.data(), hostlen, p);
+  }
+  *p++ = ':';
+  std::copy_n(serv.data(), servlen, p);
+
+  return s;
+}
+
 static int STDERR_COPY = -1;
 static int STDOUT_COPY = -1;
 

src/util.h

@@ -459,6 +459,11 @@ bool numeric_host(const char *hostname, int family);
 // failed, "unknown" is returned.
 std::string numeric_name(const struct sockaddr *sa, socklen_t salen);
 
+// Returns string representation of numeric address and port of |addr|
+// of length |salen|.  The format is like <HOST>:<PORT>.  For IPv6
+// address, address is enclosed by square brackets ([]).
+std::string numeric_hostport(const struct sockaddr *sa, socklen_t salen);
+
 // Makes internal copy of stderr (and possibly stdout in the future),
 // which is then used as pointer to /dev/stderr or /proc/self/fd/2
 void store_original_fds();