nghttpx: Allocate server id in Connection ID

bpf/reuseport_kern.c

@@ -455,6 +455,7 @@ typedef struct quic_hd {
 #define MAX_DCIDLEN 20
 #define MIN_DCIDLEN 8
 #define CID_PREFIXLEN 8
+#define CID_PREFIX_OFFSET 1
 
 enum {
   NGTCP2_PKT_INITIAL = 0x0,
@@ -579,6 +580,7 @@ int select_reuseport(struct sk_reuseport_md *reuse_md) {
   __u8 qpktbuf[6 + MAX_DCIDLEN];
   struct AES_ctx aes_ctx;
   __u8 key[AES_KEYLEN];
+  __u8 *cid_prefix;
 
   if (bpf_skb_load_bytes(reuse_md, sizeof(struct udphdr), qpktbuf,
                          sizeof(qpktbuf)) != 0) {
@@ -615,9 +617,10 @@ int select_reuseport(struct sk_reuseport_md *reuse_md) {
   case NGTCP2_PKT_INITIAL:
   case NGTCP2_PKT_0RTT:
     if (qhd.dcidlen == SV_DCIDLEN) {
-      AES_ECB_decrypt(&aes_ctx, qhd.dcid);
+      cid_prefix = qhd.dcid + CID_PREFIX_OFFSET;
+      AES_ECB_decrypt(&aes_ctx, cid_prefix);
 
-      psk_index = bpf_map_lookup_elem(&cid_prefix_map, qhd.dcid);
+      psk_index = bpf_map_lookup_elem(&cid_prefix_map, cid_prefix);
       if (psk_index != NULL) {
         sk_index = *psk_index;
 
@@ -634,9 +637,10 @@ int select_reuseport(struct sk_reuseport_md *reuse_md) {
       return SK_DROP;
     }
 
-    AES_ECB_decrypt(&aes_ctx, qhd.dcid);
+    cid_prefix = qhd.dcid + CID_PREFIX_OFFSET;
+    AES_ECB_decrypt(&aes_ctx, cid_prefix);
 
-    psk_index = bpf_map_lookup_elem(&cid_prefix_map, qhd.dcid);
+    psk_index = bpf_map_lookup_elem(&cid_prefix_map, cid_prefix);
     if (psk_index == NULL) {
       sk_index = sk_index_from_dcid(&qhd, reuse_md, *pnum_socks);
 

gennghttpxfun.py

@@ -193,6 +193,7 @@ OPTIONS = [
     "frontend-quic-require-token",
     "frontend-quic-congestion-controller",
     "frontend-quic-connection-id-encryption-key",
+    "frontend-quic-server-id",
 ]
 
 LOGVARS = [

src/shrpx.cc

@@ -1342,6 +1342,7 @@ int generate_cid_prefix(
     std::vector<std::array<uint8_t, SHRPX_QUIC_CID_PREFIXLEN>> &cid_prefixes,
     const Config *config) {
   auto &apiconf = config->api;
+  auto &quicconf = config->quic;
 
   size_t num_cid_prefix;
   if (config->single_thread) {
@@ -1360,7 +1361,8 @@ int generate_cid_prefix(
   cid_prefixes.resize(num_cid_prefix);
 
   for (auto &cid_prefix : cid_prefixes) {
-    if (create_cid_prefix(cid_prefix.data()) != 0) {
+    if (create_cid_prefix(cid_prefix.data(),
+                          quicconf.upstream.server_id.data()) != 0) {
       return -1;
     }
   }
@@ -1864,6 +1866,12 @@ void fill_default_config(Config *config) {
       assert(0);
       abort();
     }
+
+    if (RAND_bytes(upstreamconf.server_id.data(),
+                   upstreamconf.server_id.size()) != 1) {
+      assert(0);
+      abort();
+    }
   }
 
   auto &http3conf = config->http3;
@@ -3253,6 +3261,14 @@ HTTP/3 and QUIC:
               connection in a configuration  reload event, old and new
               configuration must  have this option and  share the same
               key.
+  --frontend-quic-server-id=<HEXSTRING>
+              Specify server  ID encoded in Connection  ID to identify
+              this  particular  server  instance.   Connection  ID  is
+              encrypted and  this part is  not visible in  public.  It
+              must be 2  bytes long and must be encoded  in hex string
+              (which is 4  bytes long).  If this option  is omitted, a
+              random   server  ID   is   generated   on  startup   and
+              configuration reload.
   --no-quic-bpf
               Disable eBPF.
   --frontend-http3-window-size=<SIZE>
@@ -4053,6 +4069,8 @@ int main(int argc, char **argv) {
          required_argument, &flag, 183},
         {SHRPX_OPT_FRONTEND_QUIC_CONNECTION_ID_ENCRYPTION_KEY.c_str(),
          required_argument, &flag, 184},
+        {SHRPX_OPT_FRONTEND_QUIC_SERVER_ID.c_str(), required_argument, &flag,
+         185},
         {nullptr, 0, nullptr, 0}};
 
     int option_index = 0;
@@ -4936,6 +4954,11 @@ int main(int argc, char **argv) {
             SHRPX_OPT_FRONTEND_QUIC_CONNECTION_ID_ENCRYPTION_KEY,
             StringRef{optarg});
         break;
+      case 185:
+        // --frontend-quic-server-id
+        cmdcfgs.emplace_back(SHRPX_OPT_FRONTEND_QUIC_SERVER_ID,
+                             StringRef{optarg});
+        break;
       default:
         break;
       }

src/shrpx_config.cc

@@ -2273,6 +2273,11 @@ int option_lookup_token(const char *name, size_t namelen) {
     break;
   case 23:
     switch (name[22]) {
+    case 'd':
+      if (util::strieq_l("frontend-quic-server-i", name, 22)) {
+        return SHRPX_OPTID_FRONTEND_QUIC_SERVER_ID;
+      }
+      break;
     case 'e':
       if (util::strieq_l("client-private-key-fil", name, 22)) {
         return SHRPX_OPTID_CLIENT_PRIVATE_KEY_FILE;
@@ -4036,6 +4041,17 @@ int parse_config(Config *config, int optid, const StringRef &opt,
                      optarg);
 #endif // ENABLE_HTTP3
 
+    return 0;
+  case SHRPX_OPTID_FRONTEND_QUIC_SERVER_ID:
+#ifdef ENABLE_HTTP3
+    if (optarg.size() != config->quic.upstream.server_id.size() * 2 ||
+        !util::is_hex_string(optarg)) {
+      LOG(ERROR) << opt << ": must be a hex-string";
+      return -1;
+    }
+    util::decode_hex(std::begin(config->quic.upstream.server_id), optarg);
+#endif // ENABLE_HTTP3
+
     return 0;
   case SHRPX_OPTID_CONF:
     LOG(WARN) << "conf: ignored";

src/shrpx_config.h

@@ -393,6 +393,8 @@ constexpr auto SHRPX_OPT_FRONTEND_QUIC_CONGESTION_CONTROLLER =
     StringRef::from_lit("frontend-quic-congestion-controller");
 constexpr auto SHRPX_OPT_FRONTEND_QUIC_CONNECTION_ID_ENCRYPTION_KEY =
     StringRef::from_lit("frontend-quic-connection-id-encryption-key");
+constexpr auto SHRPX_OPT_FRONTEND_QUIC_SERVER_ID =
+    StringRef::from_lit("frontend-quic-server-id");
 
 constexpr size_t SHRPX_OBFUSCATED_NODE_LENGTH = 8;
 
@@ -764,6 +766,7 @@ struct QUICConfig {
     bool early_data;
     bool require_token;
     std::array<uint8_t, SHRPX_QUIC_CID_ENCRYPTION_KEYLEN> cid_encryption_key;
+    std::array<uint8_t, SHRPX_QUIC_SERVER_IDLEN> server_id;
   } upstream;
   struct {
     StringRef prog_file;
@@ -1223,6 +1226,7 @@ enum {
   SHRPX_OPTID_FRONTEND_QUIC_IDLE_TIMEOUT,
   SHRPX_OPTID_FRONTEND_QUIC_QLOG_DIR,
   SHRPX_OPTID_FRONTEND_QUIC_REQUIRE_TOKEN,
+  SHRPX_OPTID_FRONTEND_QUIC_SERVER_ID,
   SHRPX_OPTID_FRONTEND_READ_TIMEOUT,
   SHRPX_OPTID_FRONTEND_WRITE_TIMEOUT,
   SHRPX_OPTID_HEADER_FIELD_BUFFER,

src/shrpx_connection_handler.cc

@@ -1292,9 +1292,9 @@ int ConnectionHandler::quic_ipc_read() {
 
   std::array<uint8_t, SHRPX_QUIC_DECRYPTED_DCIDLEN> decrypted_dcid;
 
-  if (decrypt_quic_connection_id(decrypted_dcid.data(), dcid,
-                                 quicconf.upstream.cid_encryption_key.data()) !=
-      0) {
+  if (decrypt_quic_connection_id(
+          decrypted_dcid.data(), dcid + SHRPX_QUIC_CID_PREFIX_OFFSET,
+          quicconf.upstream.cid_encryption_key.data()) != 0) {
     return -1;
   }
 

src/shrpx_quic.cc

@@ -143,30 +143,40 @@ int quic_send_packet(const UpstreamAddr *faddr, const sockaddr *remote_sa,
   return 0;
 }
 
-int generate_quic_connection_id(ngtcp2_cid &cid, size_t cidlen) {
+int generate_quic_retry_connection_id(ngtcp2_cid &cid, size_t cidlen,
+                                      const uint8_t *server_id,
+                                      const uint8_t *key) {
+  assert(cidlen == SHRPX_QUIC_SCIDLEN);
+
   if (RAND_bytes(cid.data, cidlen) != 1) {
     return -1;
   }
 
   cid.datalen = cidlen;
 
-  return 0;
+  auto p = cid.data + SHRPX_QUIC_CID_PREFIX_OFFSET;
+
+  std::copy_n(server_id, SHRPX_QUIC_SERVER_IDLEN, p);
+
+  return encrypt_quic_connection_id(p, p, key);
 }
 
 int generate_encrypted_quic_connection_id(ngtcp2_cid &cid, size_t cidlen,
                                           const uint8_t *cid_prefix,
                                           const uint8_t *key) {
-  assert(cidlen > SHRPX_QUIC_CID_PREFIXLEN);
+  assert(cidlen == SHRPX_QUIC_SCIDLEN);
 
-  auto p = std::copy_n(cid_prefix, SHRPX_QUIC_CID_PREFIXLEN, cid.data);
-
-  if (RAND_bytes(p, cidlen - SHRPX_QUIC_CID_PREFIXLEN) != 1) {
+  if (RAND_bytes(cid.data, cidlen) != 1) {
     return -1;
   }
 
   cid.datalen = cidlen;
 
-  return encrypt_quic_connection_id(cid.data, cid.data, key);
+  auto p = cid.data + SHRPX_QUIC_CID_PREFIX_OFFSET;
+
+  std::copy_n(cid_prefix, SHRPX_QUIC_CID_PREFIXLEN, p);
+
+  return encrypt_quic_connection_id(p, p, key);
 }
 
 int encrypt_quic_connection_id(uint8_t *dest, const uint8_t *src,

src/shrpx_quic.h

@@ -62,7 +62,10 @@ namespace shrpx {
 struct UpstreamAddr;
 
 constexpr size_t SHRPX_QUIC_SCIDLEN = 20;
+constexpr size_t SHRPX_QUIC_SERVER_IDLEN = 2;
+// SHRPX_QUIC_CID_PREFIXLEN includes SHRPX_QUIC_SERVER_IDLEN.
 constexpr size_t SHRPX_QUIC_CID_PREFIXLEN = 8;
+constexpr size_t SHRPX_QUIC_CID_PREFIX_OFFSET = 1;
 constexpr size_t SHRPX_QUIC_DECRYPTED_DCIDLEN = 16;
 constexpr size_t SHRPX_QUIC_CID_ENCRYPTION_KEYLEN = 16;
 constexpr size_t SHRPX_QUIC_MAX_UDP_PAYLOAD_SIZE = 1472;
@@ -78,7 +81,9 @@ int quic_send_packet(const UpstreamAddr *faddr, const sockaddr *remote_sa,
                      size_t local_salen, const uint8_t *data, size_t datalen,
                      size_t gso_size);
 
-int generate_quic_connection_id(ngtcp2_cid &cid, size_t cidlen);
+int generate_quic_retry_connection_id(ngtcp2_cid &cid, size_t cidlen,
+                                      const uint8_t *server_id,
+                                      const uint8_t *key);
 
 int generate_encrypted_quic_connection_id(ngtcp2_cid &cid, size_t cidlen,
                                           const uint8_t *cid_prefix,

src/shrpx_quic_connection_handler.cc

@@ -128,7 +128,7 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr,
 
     if (dcidlen == SHRPX_QUIC_SCIDLEN) {
       if (decrypt_quic_connection_id(
-              decrypted_dcid.data(), dcid,
+              decrypted_dcid.data(), dcid + SHRPX_QUIC_CID_PREFIX_OFFSET,
               quicconf.upstream.cid_encryption_key.data()) != 0) {
         return 0;
       }
@@ -419,9 +419,14 @@ int QUICConnectionHandler::send_retry(
     return -1;
   }
 
+  auto config = get_config();
+  auto &quicconf = config->quic;
+
   ngtcp2_cid retry_scid;
 
-  if (generate_quic_connection_id(retry_scid, SHRPX_QUIC_SCIDLEN) != 0) {
+  if (generate_quic_retry_connection_id(
+          retry_scid, SHRPX_QUIC_SCIDLEN, quicconf.upstream.server_id.data(),
+          quicconf.upstream.cid_encryption_key.data()) != 0) {
     return -1;
   }
 

src/shrpx_worker.cc

@@ -1273,8 +1273,10 @@ void downstream_failure(DownstreamAddr *addr, const Address *raddr) {
 }
 
 #ifdef ENABLE_HTTP3
-int create_cid_prefix(uint8_t *cid_prefix) {
-  if (RAND_bytes(cid_prefix, SHRPX_QUIC_CID_PREFIXLEN) != 1) {
+int create_cid_prefix(uint8_t *cid_prefix, const uint8_t *server_id) {
+  auto p = std::copy_n(server_id, SHRPX_QUIC_SERVER_IDLEN, cid_prefix);
+
+  if (RAND_bytes(p, SHRPX_QUIC_CID_PREFIXLEN - SHRPX_QUIC_SERVER_IDLEN) != 1) {
     return -1;
   }
 

src/shrpx_worker.h

@@ -467,8 +467,8 @@ void downstream_failure(DownstreamAddr *addr, const Address *raddr);
 #ifdef ENABLE_HTTP3
 // Creates unpredictable SHRPX_QUIC_CID_PREFIXLEN bytes sequence which
 // is used as a prefix of QUIC Connection ID.  This function returns
-// -1 on failure.
-int create_cid_prefix(uint8_t *cid_prefix);
+// -1 on failure.  |server_id| must be 2 bytes long.
+int create_cid_prefix(uint8_t *cid_prefix, const uint8_t *server_id);
 #endif // ENABLE_HTTP3
 
 } // namespace shrpx