nghttpx: Read quic packet

src/shrpx_client_handler.cc

@@ -50,6 +50,7 @@
 #include "shrpx_connect_blocker.h"
 #include "shrpx_api_downstream_connection.h"
 #include "shrpx_health_monitor_downstream_connection.h"
+#include "shrpx_http3_upstream.h"
 #include "shrpx_log.h"
 #include "util.h"
 #include "template.h"
@@ -285,6 +286,15 @@ int ClientHandler::write_tls() {
   }
 }
 
+int ClientHandler::read_quic(const UpstreamAddr *faddr,
+                             const Address &remote_addr,
+                             const Address &local_addr, const uint8_t *data,
+                             size_t datalen) {
+  auto upstream = static_cast<Http3Upstream *>(upstream_.get());
+
+  return upstream->on_read(faddr, remote_addr, local_addr, data, datalen);
+}
+
 int ClientHandler::upstream_noop() { return 0; }
 
 int ClientHandler::upstream_read() {
@@ -401,7 +411,8 @@ ClientHandler::ClientHandler(Worker *worker, int fd, SSL *ssl,
             get_config()->conn.upstream.ratelimit.write,
             get_config()->conn.upstream.ratelimit.read, writecb, readcb,
             timeoutcb, this, get_config()->tls.dyn_rec.warmup_threshold,
-            get_config()->tls.dyn_rec.idle_timeout, Proto::NONE),
+            get_config()->tls.dyn_rec.idle_timeout,
+            faddr->quic ? Proto::HTTP3 : Proto::NONE),
       ipaddr_(make_string_ref(balloc_, ipaddr)),
       port_(make_string_ref(balloc_, port)),
       faddr_(faddr),
@@ -422,14 +433,16 @@ ClientHandler::ClientHandler(Worker *worker, int fd, SSL *ssl,
 
   auto config = get_config();
 
-  if (faddr_->accept_proxy_protocol ||
-      config->conn.upstream.accept_proxy_protocol) {
-    read_ = &ClientHandler::read_clear;
-    write_ = &ClientHandler::noop;
-    on_read_ = &ClientHandler::proxy_protocol_read;
-    on_write_ = &ClientHandler::upstream_noop;
-  } else {
-    setup_upstream_io_callback();
+  if (!faddr->quic) {
+    if (faddr_->accept_proxy_protocol ||
+        config->conn.upstream.accept_proxy_protocol) {
+      read_ = &ClientHandler::read_clear;
+      write_ = &ClientHandler::noop;
+      on_read_ = &ClientHandler::proxy_protocol_read;
+      on_write_ = &ClientHandler::upstream_noop;
+    } else {
+      setup_upstream_io_callback();
+    }
   }
 
   auto &fwdconf = config->http.forwarded;
@@ -491,6 +504,12 @@ void ClientHandler::setup_upstream_io_callback() {
   }
 }
 
+void ClientHandler::setup_http3_upstream(
+    std::unique_ptr<Http3Upstream> &&upstream) {
+  upstream_ = std::move(upstream);
+  alpn_ = StringRef::from_lit("h3");
+}
+
 ClientHandler::~ClientHandler() {
   if (LOG_ENABLED(INFO)) {
     CLOG(INFO, this) << "Deleting";

src/shrpx_client_handler.h

@@ -53,6 +53,7 @@ class Downstream;
 struct WorkerStat;
 struct DownstreamAddrGroup;
 struct DownstreamAddr;
+class Http3Upstream;
 
 class ClientHandler {
 public:
@@ -70,6 +71,9 @@ public:
   int read_tls();
   int write_tls();
 
+  int read_quic(const UpstreamAddr *faddr, const Address &remote_addr,
+                const Address &local_addr, const uint8_t *data, size_t datalen);
+
   int upstream_noop();
   int upstream_read();
   int upstream_http2_connhd_read();
@@ -143,6 +147,8 @@ public:
 
   void setup_upstream_io_callback();
 
+  void setup_http3_upstream(std::unique_ptr<Http3Upstream> &&upstream);
+
   // Returns string suitable for use in "by" parameter of Forwarded
   // header field.
   StringRef get_forwarded_by() const;

src/shrpx_config.cc

@@ -2655,6 +2655,7 @@ int parse_config(Config *config, int optid, const StringRef &opt,
     addr.sni_fwd = params.sni_fwd;
     addr.alt_mode = params.alt_mode;
     addr.accept_proxy_protocol = params.proxyproto;
+    addr.quic = params.quic;
 
     if (addr.alt_mode == UpstreamAltMode::API) {
       apiconf.enabled = true;
@@ -3977,6 +3978,8 @@ StringRef strproto(Proto proto) {
     return StringRef::from_lit("http/1.1");
   case Proto::HTTP2:
     return StringRef::from_lit("h2");
+  case Proto::HTTP3:
+    return StringRef::from_lit("h3");
   case Proto::MEMCACHED:
     return StringRef::from_lit("memcached");
   }

src/shrpx_config.h

@@ -370,6 +370,7 @@ enum class Proto {
   NONE,
   HTTP1,
   HTTP2,
+  HTTP3,
   MEMCACHED,
 };
 
@@ -458,6 +459,7 @@ struct UpstreamAddr {
   bool sni_fwd;
   // true if client is supposed to send PROXY protocol v1 header.
   bool accept_proxy_protocol;
+  bool quic;
   int fd;
 };
 
@@ -698,6 +700,12 @@ struct TLSConfig {
   bool no_postpone_early_data;
 };
 
+struct QUICConfig {
+  struct {
+    std::array<uint8_t, 32> secret;
+  } stateless_reset;
+};
+
 // custom error page
 struct ErrorPage {
   // not NULL-terminated
@@ -950,6 +958,7 @@ struct Config {
         http{},
         http2{},
         tls{},
+        quic{},
         logging{},
         conn{},
         api{},
@@ -983,6 +992,7 @@ struct Config {
   HttpConfig http;
   Http2Config http2;
   TLSConfig tls;
+  QUICConfig quic;
   LoggingConfig logging;
   ConnectionConfig conn;
   APIConfig api;

src/shrpx_connection.cc

@@ -312,10 +312,13 @@ BIO_METHOD *create_bio_method() {
 void Connection::set_ssl(SSL *ssl) {
   tls.ssl = ssl;
 
-  auto &tlsconf = get_config()->tls;
-  auto bio = BIO_new(tlsconf.bio_method);
-  BIO_set_data(bio, this);
-  SSL_set_bio(tls.ssl, bio, bio);
+  if (proto != Proto::HTTP3) {
+    auto &tlsconf = get_config()->tls;
+    auto bio = BIO_new(tlsconf.bio_method);
+    BIO_set_data(bio, this);
+    SSL_set_bio(tls.ssl, bio, bio);
+  }
+
   SSL_set_app_data(tls.ssl, this);
 }
 

src/shrpx_connection_handler.cc

@@ -222,11 +222,11 @@ int ConnectionHandler::create_single_worker() {
   );
 
   quic_cert_tree_ = tls::create_cert_lookup_tree();
-  tls::setup_quic_server_ssl_context(quic_all_ssl_ctx_, quic_indexed_ssl_ctx_,
-                                     quic_cert_tree_.get()
+  auto quic_sv_ssl_ctx = tls::setup_quic_server_ssl_context(
+      quic_all_ssl_ctx_, quic_indexed_ssl_ctx_, quic_cert_tree_.get()
 #ifdef HAVE_NEVERBLEED
-                                         ,
-                                     nb_
+                                                    ,
+      nb_
 #endif // HAVE_NEVERBLEED
   );
 
@@ -261,7 +261,8 @@ int ConnectionHandler::create_single_worker() {
 
   single_worker_ = std::make_unique<Worker>(
       loop_, sv_ssl_ctx, cl_ssl_ctx, session_cache_ssl_ctx, cert_tree_.get(),
-      ticket_keys_, this, config->conn.downstream);
+      quic_sv_ssl_ctx, quic_cert_tree_.get(), ticket_keys_, this,
+      config->conn.downstream);
 #ifdef HAVE_MRUBY
   if (single_worker_->create_mruby_context() != 0) {
     return -1;
@@ -288,11 +289,11 @@ int ConnectionHandler::create_worker_thread(size_t num) {
   );
 
   quic_cert_tree_ = tls::create_cert_lookup_tree();
-  tls::setup_quic_server_ssl_context(quic_all_ssl_ctx_, quic_indexed_ssl_ctx_,
-                                     quic_cert_tree_.get()
+  auto quic_sv_ssl_ctx = tls::setup_quic_server_ssl_context(
+      quic_all_ssl_ctx_, quic_indexed_ssl_ctx_, quic_cert_tree_.get()
 #  ifdef HAVE_NEVERBLEED
-                                         ,
-                                     nb_
+                                                    ,
+      nb_
 #  endif // HAVE_NEVERBLEED
   );
 
@@ -337,7 +338,8 @@ int ConnectionHandler::create_worker_thread(size_t num) {
 
     auto worker = std::make_unique<Worker>(
         loop, sv_ssl_ctx, cl_ssl_ctx, session_cache_ssl_ctx, cert_tree_.get(),
-        ticket_keys_, this, config->conn.downstream);
+        quic_sv_ssl_ctx, quic_cert_tree_.get(), ticket_keys_, this,
+        config->conn.downstream);
 #  ifdef HAVE_MRUBY
     if (worker->create_mruby_context() != 0) {
       return -1;

src/shrpx_http3_upstream.cc

@@ -23,17 +23,201 @@
  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
 #include "shrpx_http3_upstream.h"
+
+#include <stdio.h>
+
+#include <ngtcp2/ngtcp2_crypto.h>
+
 #include "shrpx_client_handler.h"
 #include "shrpx_downstream.h"
 #include "shrpx_downstream_connection.h"
 #include "shrpx_log.h"
+#include "shrpx_quic.h"
+#include "shrpx_worker.h"
+#include "util.h"
 
 namespace shrpx {
 
 Http3Upstream::Http3Upstream(ClientHandler *handler)
-    : handler_{handler}, tls_alert_{0} {}
+    : handler_{handler},
+      conn_{nullptr},
+      last_error_{QUICErrorType::Transport, 0},
+      tls_alert_{0} {}
+
+Http3Upstream::~Http3Upstream() {
+  if (conn_) {
+    auto worker = handler_->get_worker();
+    auto quic_client_handler = worker->get_quic_connection_handler();
+
+    quic_client_handler->remove_connection_id(&initial_client_dcid_);
+
+    std::vector<ngtcp2_cid> scids(ngtcp2_conn_get_num_scid(conn_));
+    ngtcp2_conn_get_scid(conn_, scids.data());
+
+    for (auto &cid : scids) {
+      quic_client_handler->remove_connection_id(&cid);
+    }
+
+    ngtcp2_conn_del(conn_);
+  }
+}
+
+namespace {
+void log_printf(void *user_data, const char *fmt, ...) {
+  va_list ap;
+  std::array<char, 4096> buf;
+
+  va_start(ap, fmt);
+  auto nwrite = vsnprintf(buf.data(), buf.size(), fmt, ap);
+  va_end(ap);
+
+  if (nwrite >= buf.size()) {
+    nwrite = buf.size() - 1;
+  }
+
+  buf[nwrite++] = '\n';
+
+  write(fileno(stderr), buf.data(), nwrite);
+}
+} // namespace
+
+namespace {
+void rand(uint8_t *dest, size_t destlen, const ngtcp2_rand_ctx *rand_ctx) {
+  util::random_bytes(dest, dest + destlen,
+                     *static_cast<std::mt19937 *>(rand_ctx->native_handle));
+}
+} // namespace
 
-Http3Upstream::~Http3Upstream() {}
+namespace {
+int get_new_connection_id(ngtcp2_conn *conn, ngtcp2_cid *cid, uint8_t *token,
+                          size_t cidlen, void *user_data) {
+  if (generate_quic_connection_id(cid, cidlen) != 0) {
+    return NGTCP2_ERR_CALLBACK_FAILURE;
+  }
+
+  auto config = get_config();
+  auto &quicconf = config->quic;
+  auto &secret = quicconf.stateless_reset.secret;
+
+  if (generate_quic_stateless_reset_token(token, cid, secret.data(),
+                                          secret.size()) != 0) {
+    return NGTCP2_ERR_CALLBACK_FAILURE;
+  }
+
+  return 0;
+}
+} // namespace
+
+namespace {
+int remove_connection_id(ngtcp2_conn *conn, const ngtcp2_cid *cid,
+                         void *user_data) {
+  auto upstream = static_cast<Http3Upstream *>(user_data);
+  auto handler = upstream->get_client_handler();
+  auto worker = handler->get_worker();
+  auto quic_conn_handler = worker->get_quic_connection_handler();
+
+  quic_conn_handler->remove_connection_id(cid);
+
+  return 0;
+}
+} // namespace
+
+int Http3Upstream::init(const UpstreamAddr *faddr, const Address &remote_addr,
+                        const Address &local_addr,
+                        const ngtcp2_pkt_hd &initial_hd) {
+  int rv;
+
+  auto worker = handler_->get_worker();
+
+  auto callbacks = ngtcp2_callbacks{
+      nullptr, // client_initial
+      ngtcp2_crypto_recv_client_initial_cb,
+      ngtcp2_crypto_recv_crypto_data_cb,
+      nullptr, // handshake_completed
+      nullptr, // recv_version_negotiation
+      ngtcp2_crypto_encrypt_cb,
+      ngtcp2_crypto_decrypt_cb,
+      ngtcp2_crypto_hp_mask_cb,
+      nullptr, // recv_stream_data
+      nullptr, // acked_crypto_offset
+      nullptr, // acked_stream_data_offset
+      nullptr, // stream_open
+      nullptr, // stream_close
+      nullptr, // recv_stateless_reset
+      nullptr, // recv_retry
+      nullptr, // extend_max_local_streams_bidi
+      nullptr, // extend_max_local_streams_uni
+      rand,
+      get_new_connection_id,
+      remove_connection_id,
+      ngtcp2_crypto_update_key_cb,
+      nullptr, // path_validation
+      nullptr, // select_preferred_addr
+      nullptr, // stream_reset
+      nullptr, // extend_max_remote_streams_bidi
+      nullptr, // extend_max_remote_streams_uni
+      nullptr, // extend_max_stream_data
+      nullptr, // dcid_status
+      nullptr, // handshake_confirmed
+      nullptr, // recv_new_token
+      ngtcp2_crypto_delete_crypto_aead_ctx_cb,
+      ngtcp2_crypto_delete_crypto_cipher_ctx_cb,
+      nullptr, // recv_datagram
+      nullptr, // ack_datagram
+      nullptr, // lost_datagram
+      ngtcp2_crypto_get_path_challenge_data_cb,
+      nullptr, // stream_stop_sending
+  };
+
+  initial_client_dcid_ = initial_hd.dcid;
+
+  ngtcp2_cid scid;
+
+  if (generate_quic_connection_id(&scid, SHRPX_QUIC_SCIDLEN) != 0) {
+    return -1;
+  }
+
+  ngtcp2_settings settings;
+  ngtcp2_settings_default(&settings);
+  settings.log_printf = log_printf;
+  settings.initial_ts = quic_timestamp();
+  settings.cc_algo = NGTCP2_CC_ALGO_BBR;
+  settings.max_window = 6_m;
+  settings.max_stream_window = 6_m;
+  settings.max_udp_payload_size = SHRPX_MAX_UDP_PAYLOAD_SIZE;
+  settings.rand_ctx = {&worker->get_randgen()};
+
+  ngtcp2_transport_params params;
+  ngtcp2_transport_params_default(&params);
+  params.initial_max_data = 1_m;
+  params.initial_max_stream_data_bidi_remote = 256_k;
+  params.initial_max_stream_data_uni = 256_k;
+  params.max_idle_timeout = 30 * NGTCP2_SECONDS;
+  params.original_dcid = initial_hd.dcid;
+
+  auto path = ngtcp2_path{
+      {local_addr.len, const_cast<sockaddr *>(&local_addr.su.sa)},
+      {remote_addr.len, const_cast<sockaddr *>(&remote_addr.su.sa)},
+      const_cast<UpstreamAddr *>(faddr),
+  };
+
+  rv = ngtcp2_conn_server_new(&conn_, &initial_hd.scid, &scid, &path,
+                              initial_hd.version, &callbacks, &settings,
+                              &params, nullptr, this);
+  if (rv != 0) {
+    LOG(ERROR) << "ngtcp2_conn_server_new: " << ngtcp2_strerror(rv);
+    return -1;
+  }
+
+  ngtcp2_conn_set_tls_native_handle(conn_, handler_->get_ssl());
+
+  auto quic_connection_handler = worker->get_quic_connection_handler();
+
+  quic_connection_handler->add_connection_id(&initial_client_dcid_, handler_);
+  quic_connection_handler->add_connection_id(&scid, handler_);
+
+  return 0;
+}
 
 int Http3Upstream::on_read() { return 0; }
 
@@ -127,16 +311,76 @@ int Http3Upstream::on_read(const UpstreamAddr *faddr,
                            const Address &remote_addr,
                            const Address &local_addr, const uint8_t *data,
                            size_t datalen) {
+  int rv;
+  ngtcp2_pkt_info pi{};
+
+  auto path = ngtcp2_path{
+      {
+          local_addr.len,
+          const_cast<sockaddr *>(&local_addr.su.sa),
+      },
+      {
+          remote_addr.len,
+          const_cast<sockaddr *>(&remote_addr.su.sa),
+      },
+      const_cast<UpstreamAddr *>(faddr),
+  };
+
+  rv = ngtcp2_conn_read_pkt(conn_, &path, &pi, data, datalen, quic_timestamp());
+  if (rv != 0) {
+    LOG(ERROR) << "ngtcp2_conn_read_pkt: " << ngtcp2_strerror(rv);
+
+    switch (rv) {
+    case NGTCP2_ERR_DRAINING:
+      // TODO Start drain period
+      return -1;
+    case NGTCP2_ERR_RETRY:
+      // TODO Send Retry packet
+      return -1;
+    case NGTCP2_ERR_REQUIRED_TRANSPORT_PARAM:
+    case NGTCP2_ERR_MALFORMED_TRANSPORT_PARAM:
+    case NGTCP2_ERR_TRANSPORT_PARAM:
+      // If rv indicates transport_parameters related error, we should
+      // send TRANSPORT_PARAMETER_ERROR even if last_error_.code is
+      // already set.  This is because OpenSSL might set Alert.
+      last_error_ = quic_err_transport(rv);
+      break;
+    case NGTCP2_ERR_DROP_CONN:
+      return -1;
+    default:
+      if (!last_error_.code) {
+        last_error_ = quic_err_transport(rv);
+      }
+    }
+
+    // TODO Send connection close
+    return handle_error();
+  }
+
   return 0;
 }
 
+int Http3Upstream::handle_error() { return -1; }
+
 int Http3Upstream::on_rx_secret(ngtcp2_crypto_level level,
                                 const uint8_t *secret, size_t secretlen) {
+  if (ngtcp2_crypto_derive_and_install_rx_key(conn_, nullptr, nullptr, nullptr,
+                                              level, secret, secretlen) != 0) {
+    LOG(ERROR) << "ngtcp2_crypto_derive_and_install_rx_key failed";
+    return -1;
+  }
+
   return 0;
 }
 
 int Http3Upstream::on_tx_secret(ngtcp2_crypto_level level,
                                 const uint8_t *secret, size_t secretlen) {
+  if (ngtcp2_crypto_derive_and_install_tx_key(conn_, nullptr, nullptr, nullptr,
+                                              level, secret, secretlen) != 0) {
+    LOG(ERROR) << "ngtcp2_crypto_derive_and_install_tx_key failed";
+    return -1;
+  }
+
   return 0;
 }
 

src/shrpx_http3_upstream.h

@@ -30,6 +30,7 @@
 #include <ngtcp2/ngtcp2.h>
 
 #include "shrpx_upstream.h"
+#include "shrpx_quic.h"
 #include "network.h"
 
 using namespace nghttp2;
@@ -84,6 +85,9 @@ public:
   virtual bool push_enabled() const;
   virtual void cancel_premature_downstream(Downstream *promised_downstream);
 
+  int init(const UpstreamAddr *faddr, const Address &remote_addr,
+           const Address &local_addr, const ngtcp2_pkt_hd &initial_hd);
+
   int on_read(const UpstreamAddr *faddr, const Address &remote_addr,
               const Address &local_addr, const uint8_t *data, size_t datalen);
 
@@ -97,8 +101,13 @@ public:
 
   void set_tls_alert(uint8_t alert);
 
+  int handle_error();
+
 private:
   ClientHandler *handler_;
+  ngtcp2_cid initial_client_dcid_;
+  ngtcp2_conn *conn_;
+  QUICError last_error_;
   uint8_t tls_alert_;
 };
 

src/shrpx_quic.cc

@@ -29,6 +29,13 @@
 #include <netdb.h>
 
 #include <array>
+#include <chrono>
+
+#include <ngtcp2/ngtcp2_crypto.h>
+
+#include <nghttp3/nghttp3.h>
+
+#include <openssl/rand.h>
 
 #include "shrpx_config.h"
 #include "shrpx_log.h"
@@ -39,6 +46,34 @@ using namespace nghttp2;
 
 namespace shrpx {
 
+QUICError quic_err_transport(int liberr) {
+  if (liberr == NGTCP2_ERR_RECV_VERSION_NEGOTIATION) {
+    return {QUICErrorType::TransportVersionNegotiation, 0};
+  }
+  return {QUICErrorType::Transport,
+          ngtcp2_err_infer_quic_transport_error_code(liberr)};
+}
+
+QUICError quic_err_idle_timeout() {
+  return {QUICErrorType::TransportIdleTimeout, 0};
+}
+
+QUICError quic_err_tls(int alert) {
+  return {QUICErrorType::Transport,
+          static_cast<uint64_t>(NGTCP2_CRYPTO_ERROR | alert)};
+}
+
+QUICError quic_err_app(int liberr) {
+  return {QUICErrorType::Application,
+          nghttp3_err_infer_quic_app_error_code(liberr)};
+}
+
+ngtcp2_tstamp quic_timestamp() {
+  return std::chrono::duration_cast<std::chrono::nanoseconds>(
+             std::chrono::steady_clock::now().time_since_epoch())
+      .count();
+}
+
 int create_quic_server_socket(UpstreamAddr &faddr) {
   std::array<char, STRERROR_BUFSIZE> errbuf;
   int fd = -1;
@@ -192,4 +227,28 @@ int quic_send_packet(const UpstreamAddr *addr, const sockaddr *remote_sa,
   return 0;
 }
 
+int generate_quic_connection_id(ngtcp2_cid *cid, size_t cidlen) {
+  if (RAND_bytes(cid->data, cidlen) != 1) {
+    return -1;
+  }
+
+  cid->datalen = cidlen;
+
+  return 0;
+}
+
+int generate_quic_stateless_reset_token(uint8_t *token, const ngtcp2_cid *cid,
+                                        const uint8_t *secret,
+                                        size_t secretlen) {
+  ngtcp2_crypto_md md;
+  ngtcp2_crypto_md_init(&md, const_cast<EVP_MD *>(EVP_sha256()));
+
+  if (ngtcp2_crypto_generate_stateless_reset_token(token, &md, secret,
+                                                   secretlen, cid) != 0) {
+    return -1;
+  }
+
+  return 0;
+}
+
 } // namespace shrpx

src/shrpx_quic.h

@@ -29,11 +29,38 @@
 
 #include <stdint.h>
 
+#include <ngtcp2/ngtcp2.h>
+
 namespace shrpx {
 
 struct UpstreamAddr;
 
 constexpr size_t SHRPX_QUIC_SCIDLEN = 20;
+constexpr size_t SHRPX_MAX_UDP_PAYLOAD_SIZE = 1280;
+
+enum class QUICErrorType {
+  Application,
+  Transport,
+  TransportVersionNegotiation,
+  TransportIdleTimeout,
+};
+
+struct QUICError {
+  QUICError(QUICErrorType type, uint64_t code) : type{type}, code{code} {}
+
+  QUICErrorType type;
+  uint64_t code;
+};
+
+QUICError quic_err_transport(int liberr);
+
+QUICError quic_err_idle_timeout();
+
+QUICError quic_err_tls(int alert);
+
+QUICError quic_err_app(int liberr);
+
+ngtcp2_tstamp quic_timestamp();
 
 int create_quic_server_socket(UpstreamAddr &addr);
 
@@ -42,6 +69,12 @@ int quic_send_packet(const UpstreamAddr *addr, const sockaddr *remote_sa,
                      size_t local_salen, const uint8_t *data, size_t datalen,
                      size_t gso_size);
 
+int generate_quic_connection_id(ngtcp2_cid *cid, size_t cidlen);
+
+int generate_quic_stateless_reset_token(uint8_t *token, const ngtcp2_cid *cid,
+                                        const uint8_t *secret,
+                                        size_t secretlen);
+
 } // namespace shrpx
 
 #endif // SHRPX_QUIC_H

src/shrpx_quic_connection_handler.cc

@@ -45,6 +45,12 @@ std::string make_cid_key(const uint8_t *dcid, size_t dcidlen) {
 }
 } // namespace
 
+namespace {
+std::string make_cid_key(const ngtcp2_cid *cid) {
+  return make_cid_key(cid->data, cid->datalen);
+}
+} // namespace
+
 int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr,
                                          const Address &remote_addr,
                                          const Address &local_addr,
@@ -67,6 +73,8 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr,
 
   auto dcid_key = make_cid_key(dcid, dcidlen);
 
+  ClientHandler *handler;
+
   auto it = connections_.find(dcid_key);
   if (it == std::end(connections_)) {
     // new connection
@@ -87,18 +95,66 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr,
       return 0;
     }
 
-    return 0;
+    handler = handle_new_connection(faddr, remote_addr, local_addr, hd);
+    if (handler == nullptr) {
+      return 0;
+    }
+  } else {
+    handler = (*it).second;
   }
 
-  auto h = (*it).second.get();
-  auto upstream = static_cast<Http3Upstream *>(h->get_upstream());
-
-  // TODO Delete h on failure
-  upstream->on_read(faddr, remote_addr, local_addr, data, datalen);
+  if (handler->read_quic(faddr, remote_addr, local_addr, data, datalen) != 0) {
+    delete handler;
+  }
 
   return 0;
 }
 
+ClientHandler *QUICConnectionHandler::handle_new_connection(
+    const UpstreamAddr *faddr, const Address &remote_addr,
+    const Address &local_addr, const ngtcp2_pkt_hd &hd) {
+  std::array<char, NI_MAXHOST> host;
+  std::array<char, NI_MAXSERV> service;
+  int rv;
+
+  rv = getnameinfo(&remote_addr.su.sa, remote_addr.len, host.data(),
+                   host.size(), service.data(), service.size(),
+                   NI_NUMERICHOST | NI_NUMERICSERV);
+  if (rv != 0) {
+    LOG(ERROR) << "getnameinfo() failed: " << gai_strerror(rv);
+
+    return nullptr;
+  }
+
+  auto ssl_ctx = worker_->get_quic_sv_ssl_ctx();
+
+  assert(ssl_ctx);
+
+  auto ssl = tls::create_ssl(ssl_ctx);
+  if (ssl == nullptr) {
+    return nullptr;
+  }
+
+  // Disable TLS session ticket if we don't have working ticket
+  // keys.
+  if (!worker_->get_ticket_keys()) {
+    SSL_set_options(ssl, SSL_OP_NO_TICKET);
+  }
+
+  auto handler = std::make_unique<ClientHandler>(
+      worker_, faddr->fd, ssl, StringRef{host.data()},
+      StringRef{service.data()}, remote_addr.su.sa.sa_family, faddr);
+
+  auto upstream = std::make_unique<Http3Upstream>(handler.get());
+  if (upstream->init(faddr, remote_addr, local_addr, hd) != 0) {
+    return nullptr;
+  }
+
+  handler->setup_http3_upstream(std::move(upstream));
+
+  return handler.release();
+}
+
 namespace {
 uint32_t generate_reserved_version(const Address &addr, uint32_t version) {
   uint32_t h = 0x811C9DC5u;
@@ -154,4 +210,15 @@ int QUICConnectionHandler::send_version_negotiation(
                           0);
 }
 
+void QUICConnectionHandler::add_connection_id(const ngtcp2_cid *cid,
+                                              ClientHandler *handler) {
+  auto key = make_cid_key(cid);
+  connections_.emplace(key, handler);
+}
+
+void QUICConnectionHandler::remove_connection_id(const ngtcp2_cid *cid) {
+  auto key = make_cid_key(cid);
+  connections_.erase(key);
+}
+
 } // namespace shrpx

src/shrpx_quic_connection_handler.h

@@ -31,6 +31,8 @@
 #include <unordered_map>
 #include <string>
 
+#include <ngtcp2/ngtcp2.h>
+
 #include "network.h"
 
 using namespace nghttp2;
@@ -53,10 +55,16 @@ public:
                                const uint8_t *scid, size_t scidlen,
                                const Address &remote_addr,
                                const Address &local_addr);
+  ClientHandler *handle_new_connection(const UpstreamAddr *faddr,
+                                       const Address &remote_addr,
+                                       const Address &local_addr,
+                                       const ngtcp2_pkt_hd &hd);
+  void add_connection_id(const ngtcp2_cid *cid, ClientHandler *handler);
+  void remove_connection_id(const ngtcp2_cid *cid);
 
 private:
   Worker *worker_;
-  std::unordered_map<std::string, std::shared_ptr<ClientHandler>> connections_;
+  std::unordered_map<std::string, ClientHandler *> connections_;
 };
 
 } // namespace shrpx

src/shrpx_quic_listener.cc

@@ -37,7 +37,7 @@ void readcb(struct ev_loop *loop, ev_io *w, int revent) {
 } // namespace
 
 QUICListener::QUICListener(const UpstreamAddr *faddr, Worker *worker)
-    : faddr_(faddr), worker_(worker) {
+    : faddr_{faddr}, worker_{worker} {
   ev_io_init(&rev_, readcb, faddr_->fd, EV_READ);
   rev_.data = this;
   ev_io_start(worker_->get_loop(), &rev_);
@@ -62,6 +62,8 @@ void QUICListener::on_read() {
   uint8_t msg_ctrl[CMSG_SPACE(sizeof(in6_pktinfo))];
   msg.msg_control = msg_ctrl;
 
+  auto quic_conn_handler = worker_->get_quic_connection_handler();
+
   for (; pktcnt < 10;) {
     msg.msg_namelen = sizeof(su);
     msg.msg_controllen = sizeof(msg_ctrl);
@@ -91,6 +93,13 @@ void QUICListener::on_read() {
     if (nread == 0) {
       continue;
     }
+
+    Address remote_addr;
+    remote_addr.su = su;
+    remote_addr.len = msg.msg_namelen;
+
+    quic_conn_handler->handle_packet(faddr_, remote_addr, local_addr,
+                                     buf.data(), nread);
   }
 }
 

src/shrpx_worker.cc

@@ -131,7 +131,8 @@ create_downstream_key(const std::shared_ptr<SharedDownstreamAddr> &shared_addr,
 
 Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx,
                SSL_CTX *tls_session_cache_memcached_ssl_ctx,
-               tls::CertLookupTree *cert_tree,
+               tls::CertLookupTree *cert_tree, SSL_CTX *quic_sv_ssl_ctx,
+               tls::CertLookupTree *quic_cert_tree,
                const std::shared_ptr<TicketKeys> &ticket_keys,
                ConnectionHandler *conn_handler,
                std::shared_ptr<DownstreamConfig> downstreamconf)
@@ -144,6 +145,9 @@ Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx,
       cl_ssl_ctx_(cl_ssl_ctx),
       cert_tree_(cert_tree),
       conn_handler_(conn_handler),
+      quic_sv_ssl_ctx_{quic_sv_ssl_ctx},
+      quic_cert_tree_{quic_cert_tree},
+      quic_conn_handler_{this},
       ticket_keys_(ticket_keys),
       connect_blocker_(
           std::make_unique<ConnectBlocker>(randgen_, loop_, nullptr, nullptr)),
@@ -505,6 +509,10 @@ void Worker::process_events() {
 
 tls::CertLookupTree *Worker::get_cert_lookup_tree() const { return cert_tree_; }
 
+tls::CertLookupTree *Worker::get_quic_cert_lookup_tree() const {
+  return quic_cert_tree_;
+}
+
 std::shared_ptr<TicketKeys> Worker::get_ticket_keys() {
 #ifdef HAVE_ATOMIC_STD_SHARED_PTR
   return std::atomic_load_explicit(&ticket_keys_, std::memory_order_acquire);
@@ -535,6 +543,8 @@ SSL_CTX *Worker::get_sv_ssl_ctx() const { return sv_ssl_ctx_; }
 
 SSL_CTX *Worker::get_cl_ssl_ctx() const { return cl_ssl_ctx_; }
 
+SSL_CTX *Worker::get_quic_sv_ssl_ctx() const { return quic_sv_ssl_ctx_; }
+
 void Worker::set_graceful_shutdown(bool f) { graceful_shutdown_ = f; }
 
 bool Worker::get_graceful_shutdown() const { return graceful_shutdown_; }
@@ -579,6 +589,10 @@ ConnectionHandler *Worker::get_connection_handler() const {
   return conn_handler_;
 }
 
+QUICConnectionHandler *Worker::get_quic_connection_handler() {
+  return &quic_conn_handler_;
+}
+
 DNSTracker *Worker::get_dns_tracker() { return &dns_tracker_; }
 
 int Worker::setup_quic_server_socket() {

src/shrpx_worker.h

@@ -50,6 +50,7 @@
 #include "shrpx_live_check.h"
 #include "shrpx_connect_blocker.h"
 #include "shrpx_dns_tracker.h"
+#include "shrpx_quic_connection_handler.h"
 #include "allocator.h"
 
 using namespace nghttp2;
@@ -268,7 +269,8 @@ class Worker {
 public:
   Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx,
          SSL_CTX *tls_session_cache_memcached_ssl_ctx,
-         tls::CertLookupTree *cert_tree,
+         tls::CertLookupTree *cert_tree, SSL_CTX *quic_sv_ssl_ctx,
+         tls::CertLookupTree *quic_cert_tree,
          const std::shared_ptr<TicketKeys> &ticket_keys,
          ConnectionHandler *conn_handler,
          std::shared_ptr<DownstreamConfig> downstreamconf);
@@ -279,6 +281,7 @@ public:
   void send(const WorkerEvent &event);
 
   tls::CertLookupTree *get_cert_lookup_tree() const;
+  tls::CertLookupTree *get_quic_cert_lookup_tree() const;
 
   // These 2 functions make a lock m_ to get/set ticket keys
   // atomically.
@@ -289,6 +292,7 @@ public:
   struct ev_loop *get_loop() const;
   SSL_CTX *get_sv_ssl_ctx() const;
   SSL_CTX *get_cl_ssl_ctx() const;
+  SSL_CTX *get_quic_sv_ssl_ctx() const;
 
   void set_graceful_shutdown(bool f);
   bool get_graceful_shutdown() const;
@@ -318,6 +322,8 @@ public:
 
   ConnectionHandler *get_connection_handler() const;
 
+  QUICConnectionHandler *get_quic_connection_handler();
+
   DNSTracker *get_dns_tracker();
 
   int setup_quic_server_socket();
@@ -352,6 +358,10 @@ private:
   SSL_CTX *cl_ssl_ctx_;
   tls::CertLookupTree *cert_tree_;
   ConnectionHandler *conn_handler_;
+  SSL_CTX *quic_sv_ssl_ctx_;
+  tls::CertLookupTree *quic_cert_tree_;
+
+  QUICConnectionHandler quic_conn_handler_;
 
 #ifndef HAVE_ATOMIC_STD_SHARED_PTR
   std::mutex ticket_keys_m_;