nghttpx: Create quic server socket

src/Makefile.am

@@ -152,6 +152,7 @@ NGHTTPX_SRCS = \
 	shrpx_dns_resolver.cc shrpx_dns_resolver.h \
 	shrpx_dual_dns_resolver.cc shrpx_dual_dns_resolver.h \
 	shrpx_dns_tracker.cc shrpx_dns_tracker.h \
+	shrpx_quic.cc shrpx_quic.h \
 	buffer.h memchunk.h template.h allocator.h \
 	xsi_strerror.c xsi_strerror.h
 

src/shrpx_config.cc

@@ -785,6 +785,7 @@ struct UpstreamParams {
   bool tls;
   bool sni_fwd;
   bool proxyproto;
+  bool quic;
 };
 
 namespace {
@@ -819,6 +820,8 @@ int parse_upstream_params(UpstreamParams &out, const StringRef &src_params) {
       out.alt_mode = UpstreamAltMode::HEALTHMON;
     } else if (util::strieq_l("proxyproto", param)) {
       out.proxyproto = true;
+    } else if (util::strieq_l("quic", param)) {
+      out.quic = true;
     } else if (!param.empty()) {
       LOG(ERROR) << "frontend: " << param << ": unknown keyword";
       return -1;
@@ -2624,7 +2627,6 @@ int parse_config(Config *config, int optid, const StringRef &opt,
     return 0;
   }
   case SHRPX_OPTID_FRONTEND: {
-    auto &listenerconf = config->conn.listener;
     auto &apiconf = config->api;
 
     auto addr_end = std::find(std::begin(optarg), std::end(optarg), ';');
@@ -2642,6 +2644,11 @@ int parse_config(Config *config, int optid, const StringRef &opt,
       return -1;
     }
 
+    if (params.quic && params.alt_mode != UpstreamAltMode::NONE) {
+      LOG(ERROR) << "frontend: api or healthmon cannot be used with quic";
+      return -1;
+    }
+
     UpstreamAddr addr{};
     addr.fd = -1;
     addr.tls = params.tls;
@@ -2653,12 +2660,15 @@ int parse_config(Config *config, int optid, const StringRef &opt,
       apiconf.enabled = true;
     }
 
+    auto &addrs = params.quic ? config->conn.quic_listener.addrs
+                              : config->conn.listener.addrs;
+
     if (util::istarts_with(optarg, SHRPX_UNIX_PATH_PREFIX)) {
       auto path = std::begin(optarg) + SHRPX_UNIX_PATH_PREFIX.size();
       addr.host = make_string_ref(config->balloc, StringRef{path, addr_end});
       addr.host_unix = true;
 
-      listenerconf.addrs.push_back(std::move(addr));
+      addrs.push_back(std::move(addr));
 
       return 0;
     }
@@ -2673,21 +2683,21 @@ int parse_config(Config *config, int optid, const StringRef &opt,
 
     if (util::numeric_host(host, AF_INET)) {
       addr.family = AF_INET;
-      listenerconf.addrs.push_back(std::move(addr));
+      addrs.push_back(std::move(addr));
       return 0;
     }
 
     if (util::numeric_host(host, AF_INET6)) {
       addr.family = AF_INET6;
-      listenerconf.addrs.push_back(std::move(addr));
+      addrs.push_back(std::move(addr));
       return 0;
     }
 
     addr.family = AF_INET;
-    listenerconf.addrs.push_back(addr);
+    addrs.push_back(addr);
 
     addr.family = AF_INET6;
-    listenerconf.addrs.push_back(std::move(addr));
+    addrs.push_back(std::move(addr));
 
     return 0;
   }

src/shrpx_config.h

@@ -903,6 +903,10 @@ struct ConnectionConfig {
     int fastopen;
   } listener;
 
+  struct {
+    std::vector<UpstreamAddr> addrs;
+  } quic_listener;
+
   struct {
     struct {
       ev_tstamp http2_read;

src/shrpx_connection_handler.cc

@@ -244,6 +244,9 @@ int ConnectionHandler::create_single_worker() {
     return -1;
   }
 #endif // HAVE_MRUBY
+  if (single_worker_->setup_quic_server_socket() != 0) {
+    return -1;
+  }
 
   return 0;
 }
@@ -305,6 +308,9 @@ int ConnectionHandler::create_worker_thread(size_t num) {
       return -1;
     }
 #  endif // HAVE_MRUBY
+    if (worker->setup_quic_server_socket() != 0) {
+      return -1;
+    }
 
     workers_.push_back(std::move(worker));
     worker_loops_.push_back(loop);

src/shrpx_quic.cc

+/*
+ * nghttp2 - HTTP/2 C Library
+ *
+ * Copyright (c) 2021 Tatsuhiro Tsujikawa
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "shrpx_quic.h"
+
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <netdb.h>
+
+#include <array>
+
+#include "shrpx_config.h"
+#include "shrpx_log.h"
+#include "util.h"
+#include "xsi_strerror.h"
+
+using namespace nghttp2;
+
+namespace shrpx {
+
+int create_quic_server_socket(UpstreamAddr &faddr) {
+  std::array<char, STRERROR_BUFSIZE> errbuf;
+  int fd = -1;
+  int rv;
+
+  auto service = util::utos(faddr.port);
+  addrinfo hints{};
+  hints.ai_family = faddr.family;
+  hints.ai_socktype = SOCK_DGRAM;
+  hints.ai_flags = AI_PASSIVE;
+#ifdef AI_ADDRCONFIG
+  hints.ai_flags |= AI_ADDRCONFIG;
+#endif // AI_ADDRCONFIG
+
+  auto node =
+      faddr.host == StringRef::from_lit("*") ? nullptr : faddr.host.c_str();
+
+  addrinfo *res, *rp;
+  rv = getaddrinfo(node, service.c_str(), &hints, &res);
+#ifdef AI_ADDRCONFIG
+  if (rv != 0) {
+    // Retry without AI_ADDRCONFIG
+    hints.ai_flags &= ~AI_ADDRCONFIG;
+    rv = getaddrinfo(node, service.c_str(), &hints, &res);
+  }
+#endif // AI_ADDRCONFIG
+  if (rv != 0) {
+    LOG(FATAL) << "Unable to get IPv" << (faddr.family == AF_INET ? "4" : "6")
+               << " address for " << faddr.host << ", port " << faddr.port
+               << ": " << gai_strerror(rv);
+    return -1;
+  }
+
+  auto res_d = defer(freeaddrinfo, res);
+
+  std::array<char, NI_MAXHOST> host;
+
+  for (rp = res; rp; rp = rp->ai_next) {
+    rv = getnameinfo(rp->ai_addr, rp->ai_addrlen, host.data(), host.size(),
+                     nullptr, 0, NI_NUMERICHOST);
+    if (rv != 0) {
+      LOG(WARN) << "getnameinfo() failed: " << gai_strerror(rv);
+      continue;
+    }
+
+#ifdef SOCK_NONBLOCK
+    fd = socket(rp->ai_family, rp->ai_socktype | SOCK_NONBLOCK | SOCK_CLOEXEC,
+                rp->ai_protocol);
+    if (fd == -1) {
+      auto error = errno;
+      LOG(WARN) << "socket() syscall failed: "
+                << xsi_strerror(error, errbuf.data(), errbuf.size());
+      continue;
+    }
+#else  // !SOCK_NONBLOCK
+    fd = socket(rp->ai_family, rp->ai_socktype, rp->ai_protocol);
+    if (fd == -1) {
+      auto error = errno;
+      LOG(WARN) << "socket() syscall failed: "
+                << xsi_strerror(error, errbuf.data(), errbuf.size());
+      continue;
+    }
+    util::make_socket_nonblocking(fd);
+    util::make_socket_closeonexec(fd);
+#endif // !SOCK_NONBLOCK
+
+    int val = 1;
+    if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &val,
+                   static_cast<socklen_t>(sizeof(val))) == -1) {
+      auto error = errno;
+      LOG(WARN) << "Failed to set SO_REUSEADDR option to listener socket: "
+                << xsi_strerror(error, errbuf.data(), errbuf.size());
+      close(fd);
+      continue;
+    }
+
+    if (setsockopt(fd, SOL_SOCKET, SO_REUSEPORT, &val,
+                   static_cast<socklen_t>(sizeof(val))) == -1) {
+      auto error = errno;
+      LOG(WARN) << "Failed to set SO_REUSEPORT option to listener socket: "
+                << xsi_strerror(error, errbuf.data(), errbuf.size());
+      close(fd);
+      continue;
+    }
+
+    if (faddr.family == AF_INET6) {
+#ifdef IPV6_V6ONLY
+      if (setsockopt(fd, IPPROTO_IPV6, IPV6_V6ONLY, &val,
+                     static_cast<socklen_t>(sizeof(val))) == -1) {
+        auto error = errno;
+        LOG(WARN) << "Failed to set IPV6_V6ONLY option to listener socket: "
+                  << xsi_strerror(error, errbuf.data(), errbuf.size());
+        close(fd);
+        continue;
+      }
+#endif // IPV6_V6ONLY
+
+      if (setsockopt(fd, IPPROTO_IPV6, IPV6_RECVPKTINFO, &val,
+                     static_cast<socklen_t>(sizeof(val))) == -1) {
+        auto error = errno;
+        LOG(WARN)
+            << "Failed to set IPV6_RECVPKTINFO option to listener socket: "
+            << xsi_strerror(error, errbuf.data(), errbuf.size());
+        close(fd);
+        continue;
+      }
+    } else {
+      if (setsockopt(fd, IPPROTO_IP, IP_PKTINFO, &val,
+                     static_cast<socklen_t>(sizeof(val))) == -1) {
+        auto error = errno;
+        LOG(WARN) << "Failed to set IP_PKTINFO option to listener socket: "
+                  << xsi_strerror(error, errbuf.data(), errbuf.size());
+        close(fd);
+        continue;
+      }
+    }
+
+    // TODO Enable ECN
+
+    if (bind(fd, rp->ai_addr, rp->ai_addrlen) == -1) {
+      auto error = errno;
+      LOG(WARN) << "bind() syscall failed: "
+                << xsi_strerror(error, errbuf.data(), errbuf.size());
+      close(fd);
+      continue;
+    }
+
+    break;
+  }
+
+  if (!rp) {
+    LOG(FATAL) << "Listening " << (faddr.family == AF_INET ? "IPv4" : "IPv6")
+               << " socket failed";
+
+    return -1;
+  }
+
+  faddr.fd = fd;
+  faddr.hostport = util::make_http_hostport(mod_config()->balloc,
+                                            StringRef{host.data()}, faddr.port);
+
+  LOG(NOTICE) << "Listening on " << faddr.hostport << ", quic";
+
+  return 0;
+}
+
+} // namespace shrpx

src/shrpx_quic.h

+/*
+ * nghttp2 - HTTP/2 C Library
+ *
+ * Copyright (c) 2021 Tatsuhiro Tsujikawa
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+#ifndef SHRPX_QUIC_H
+#define SHRPX_QUIC_H
+
+#include "shrpx.h"
+
+struct UpstreamAddr;
+
+namespace shrpx {
+
+int create_quic_server_socket(UpstreamAddr &addr);
+
+} // namespace shrpx
+
+#endif // SHRPX_QUIC_H

src/shrpx_worker.cc

@@ -39,6 +39,7 @@
 #ifdef HAVE_MRUBY
 #  include "shrpx_mruby.h"
 #endif // HAVE_MRUBY
+#include "shrpx_quic.h"
 #include "util.h"
 #include "template.h"
 
@@ -136,6 +137,7 @@ Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx,
     : randgen_(util::make_mt19937()),
       worker_stat_{},
       dns_tracker_(loop),
+      quic_upstream_addrs_{get_config()->conn.quic_listener.addrs},
       loop_(loop),
       sv_ssl_ctx_(sv_ssl_ctx),
       cl_ssl_ctx_(cl_ssl_ctx),
@@ -578,6 +580,22 @@ ConnectionHandler *Worker::get_connection_handler() const {
 
 DNSTracker *Worker::get_dns_tracker() { return &dns_tracker_; }
 
+int Worker::setup_quic_server_socket() {
+  for (auto &addr : quic_upstream_addrs_) {
+    if (addr.host_unix) {
+      /* TODO Not implemented */
+      assert(0);
+      continue;
+    }
+
+    if (create_quic_server_socket(addr) != 0) {
+      return -1;
+    }
+  }
+
+  return 0;
+}
+
 namespace {
 size_t match_downstream_addr_group_host(
     const RouterConfig &routerconf, const StringRef &host,

src/shrpx_worker.h

@@ -319,6 +319,8 @@ public:
 
   DNSTracker *get_dns_tracker();
 
+  int setup_quic_server_socket();
+
 private:
 #ifndef NOTHREADS
   std::future<void> fut_;
@@ -333,6 +335,8 @@ private:
   WorkerStat worker_stat_;
   DNSTracker dns_tracker_;
 
+  std::vector<UpstreamAddr> quic_upstream_addrs_;
+
   std::shared_ptr<DownstreamConfig> downstreamconf_;
   std::unique_ptr<MemcachedDispatcher> session_cache_memcached_dispatcher_;
 #ifdef HAVE_MRUBY