nghttp2_hd: Implement local header table size limit for encoder

lib/includes/nghttp2/nghttp2.h

@@ -389,6 +389,14 @@ typedef enum {
  * The SETTINGS ID.
  */
 typedef enum {
+  /**
+   * SETTINGS_HEADER_TABLE_SIZE
+   */
+  NGHTTP2_SETTINGS_HEADER_TABLE_SIZE = 1,
+  /**
+   * SETTINGS_ENABLE_PUSH
+   */
+  NGHTTP2_SETTINGS_ENABLE_PUSH = 2,
   /**
    * SETTINGS_MAX_CONCURRENT_STREAMS
    */

lib/nghttp2_hd.h

@@ -38,6 +38,11 @@
 #define NGHTTP2_HD_MAX_ENTRY_SIZE 3072
 #define NGHTTP2_HD_ENTRY_OVERHEAD 32
 
+/* Default size of maximum table buffer size for encoder. Even if
+   remote decoder notifies larger buffer size for its decoding,
+   encoder only uses the memory up to this value. */
+#define NGHTTP2_HD_DEFAULT_LOCAL_MAX_BUFFER_SIZE (1 << 12)
+
 typedef enum {
   NGHTTP2_HD_SIDE_REQUEST = 0,
   NGHTTP2_HD_SIDE_RESPONSE = 1
@@ -85,6 +90,24 @@ typedef struct {
 typedef struct {
   /* dynamic header table */
   nghttp2_hd_ringbuf hd_table;
+  /* The header table size for decoding. If the context is initialized
+     as encoder, this value is advertised by remote endpoint
+     decoder. */
+  size_t hd_table_bufsize_max;
+  /* The current effective header table size for encoding. This value
+     is meaningful iff this context is initialized as
+     encoder. |local_hd_table_bufsize| <= |hd_table_bufsize| must be
+     hold. */
+  size_t local_hd_table_bufsize;
+  /* The maximum effective header table for encoding. Although header
+     table size is bounded by |hd_table_bufsize_max|, the encoder can
+     use smaller buffer by not retaining the header name/values beyond
+     the |local_hd_table_bufsize_max| and not referencing those
+     entries. This value is meaningful iff this context is initialized
+     as encoder. */
+  size_t local_hd_table_bufsize_max;
+  /* The number of effective entry in |hd_table|. */
+  size_t local_hd_tablelen;
   /* Holding emitted entry in deflating header block to retain
      reference count. */
   nghttp2_hd_entry **emit_set;
@@ -105,8 +128,6 @@ typedef struct {
   /* NGHTTP2_HD_SIDE_REQUEST for processing request, otherwise
      response. */
   nghttp2_hd_side side;
-  /* Maximum header table size */
-  size_t hd_table_bufsize_max;
   /* Keep track of allocated buffers in inflation */
   uint8_t **buf_track;
   /* The capacity of |buf_track| */
@@ -137,6 +158,11 @@ void nghttp2_hd_entry_free(nghttp2_hd_entry *ent);
 /*
  * Initializes |deflater| for deflating name/values pairs.
  *
+ * The encoder only uses up to
+ * NGHTTP2_HD_DEFAULT_LOCAL_MAX_BUFFER_SIZE bytes for header table
+ * even if the larger value is specified later in
+ * nghttp2_hd_change_table_size().
+ *
  * This function returns 0 if it succeeds, or one of the following
  * negative error codes:
  *
@@ -146,9 +172,22 @@ void nghttp2_hd_entry_free(nghttp2_hd_entry *ent);
 int nghttp2_hd_deflate_init(nghttp2_hd_context *deflater,
                             nghttp2_hd_side side);
 
+/*
+ * Initializes |deflater| for deflating name/values pairs.
+ *
+ * The encoder only uses up to |local_hd_table_bufsize_max| bytes for
+ * header table even if the larger value is specified later in
+ * nghttp2_hd_change_table_size().
+ *
+ * This function returns 0 if it succeeds, or one of the following
+ * negative error codes:
+ *
+ * NGHTTP2_ERR_NOMEM
+ *     Out of memory.
+ */
 int nghttp2_hd_deflate_init2(nghttp2_hd_context *deflater,
                              nghttp2_hd_side side,
-                             size_t hd_table_bufsize_max);
+                             size_t local_hd_table_bufsize_max);
 
 /*
  * Initializes |inflater| for inflating name/values pairs.
@@ -162,10 +201,6 @@ int nghttp2_hd_deflate_init2(nghttp2_hd_context *deflater,
 int nghttp2_hd_inflate_init(nghttp2_hd_context *inflater,
                             nghttp2_hd_side side);
 
-int nghttp2_hd_inflate_init2(nghttp2_hd_context *inflater,
-                             nghttp2_hd_side side,
-                             size_t hd_table_bufsize_max);
-
 /*
  * Deallocates any resources allocated for |deflater|.
  */
@@ -176,6 +211,20 @@ void nghttp2_hd_deflate_free(nghttp2_hd_context *deflater);
  */
 void nghttp2_hd_inflate_free(nghttp2_hd_context *inflater);
 
+
+/*
+ * Changes header table size in |context|. This may trigger eviction
+ * in the dynamic table.
+ *
+ * This function returns 0 if it succeeds, or one of the following
+ * negative error codes:
+ *
+ * NGHTTP2_ERR_NOMEM
+ *     Out of memory.
+ */
+int nghttp2_hd_change_table_size(nghttp2_hd_context *context,
+                                 size_t hd_table_bufsize_max);
+
 /*
  * Deflates the |nva|, which has the |nvlen| name/value pairs, into
  * the buffer pointed by the |*buf_ptr| with the length |*buflen_ptr|.

tests/main.c

@@ -231,6 +231,8 @@ int main(int argc, char* argv[])
                    test_nghttp2_hd_deflate_same_indexed_repr) ||
       !CU_add_test(pSuite, "hd_deflate_common_header_eviction",
                    test_nghttp2_hd_deflate_common_header_eviction) ||
+      !CU_add_test(pSuite, "hd_deflate_local_buffer",
+                   test_nghttp2_hd_deflate_local_buffer) ||
       !CU_add_test(pSuite, "hd_inflate_indname_inc",
                    test_nghttp2_hd_inflate_indname_inc) ||
       !CU_add_test(pSuite, "hd_inflate_indname_inc_eviction",
@@ -239,6 +241,8 @@ int main(int argc, char* argv[])
                    test_nghttp2_hd_inflate_newname_inc) ||
       !CU_add_test(pSuite, "hd_inflate_clearall_inc",
                    test_nghttp2_hd_inflate_clearall_inc) ||
+      !CU_add_test(pSuite, "hd_change_table_size",
+                   test_nghttp2_hd_change_table_size) ||
       !CU_add_test(pSuite, "hd_deflate_inflate",
                    test_nghttp2_hd_deflate_inflate) ||
       !CU_add_test(pSuite, "gzip_inflate", test_nghttp2_gzip_inflate) ||

tests/nghttp2_hd_test.c

@@ -247,6 +247,197 @@ void test_nghttp2_hd_deflate_common_header_eviction(void)
   nghttp2_hd_deflate_free(&deflater);
 }
 
+void test_nghttp2_hd_deflate_local_buffer(void)
+{
+  nghttp2_hd_context deflater, inflater;
+  size_t i;
+  ssize_t rv, blocklen;
+  uint8_t *buf = NULL;
+  size_t buflen = 0;
+  nghttp2_nv nva1[] = { MAKE_NV("k1", "v1"), /* 36 */
+                        MAKE_NV("k10", "v10"), /* 38 */
+                        MAKE_NV("k100", "v100"), /* 40 */
+                        MAKE_NV("k1000", "v1000") /* 42 */
+  }; /* Total: 156 */
+  nghttp2_nv nva2[] = { MAKE_NV("k10", "v10"), /* 38 */
+                        MAKE_NV("k1", "v1") /* 36 */
+  };
+  nghttp2_nv nv3;
+  uint8_t val[256];
+  nghttp2_nv nva4[] = { MAKE_NV(":method", "GET"),
+                        MAKE_NV(":scheme", "http")
+  };
+  nghttp2_nv *resnva;
+  nghttp2_hd_entry *ent;
+
+  memset(val, 'a', sizeof(val));
+  nv3.name = nv3.value = val;
+  nv3.namelen = nv3.valuelen = sizeof(val);
+
+  /* Check the case where entry from static table is inserted to
+     dynamic header table. And it is out of local header table
+     size. */
+  nghttp2_hd_deflate_init2(&deflater, NGHTTP2_HD_SIDE_REQUEST, 32);
+  nghttp2_hd_inflate_init(&inflater, NGHTTP2_HD_SIDE_REQUEST);
+  blocklen = nghttp2_hd_deflate_hd(&deflater, &buf, &buflen, 0,
+                                   nva4, ARRLEN(nva4));
+  CU_ASSERT(blocklen > 0);
+  /* Now header table should look like this:
+   *
+   *  0: :scheme, http (-)
+   *  1: :method, GET (-)
+   *
+   * name/value of all entries must be NULL.
+   */
+  CU_ASSERT(2 == deflater.hd_table.len);
+  CU_ASSERT(0 == deflater.local_hd_tablelen);
+  CU_ASSERT(0 == deflater.local_hd_table_bufsize);
+  for(i = 0; i < 2; ++i) {
+    ent = nghttp2_hd_table_get(&deflater, i);
+    CU_ASSERT(ent->nv.name == NULL);
+    CU_ASSERT(ent->nv.value == NULL);
+    CU_ASSERT(0 == (ent->flags & NGHTTP2_HD_FLAG_REFSET));
+  }
+
+  rv = nghttp2_hd_inflate_hd(&inflater, &resnva, buf, blocklen);
+  CU_ASSERT(2 == rv);
+  assert_nv_equal(nva4, resnva, 2);
+  nghttp2_hd_end_headers(&inflater);
+  nghttp2_nv_array_del(resnva);
+
+  nghttp2_hd_deflate_free(&deflater);
+  nghttp2_hd_inflate_free(&inflater);
+
+  /* 156 buffer size can hold all headers in local region */
+  nghttp2_hd_deflate_init2(&deflater, NGHTTP2_HD_SIDE_REQUEST,
+                           156);
+  blocklen = nghttp2_hd_deflate_hd(&deflater, &buf, &buflen, 0,
+                                   nva1, ARRLEN(nva1));
+  CU_ASSERT(blocklen > 0);
+  /* Now header table should look like this:
+   *
+   *  0: k1000, v100
+   *  1: k100, v100
+   *  2: k10, v10
+   *  3: k1, v1
+   */
+  CU_ASSERT(4 == deflater.hd_table.len);
+  CU_ASSERT(4 == deflater.local_hd_tablelen);
+  CU_ASSERT(156 == deflater.local_hd_table_bufsize);
+  for(i = 0; i < 4; ++i) {
+    CU_ASSERT(nghttp2_hd_table_get(&deflater, i)->nv.name != NULL);
+    CU_ASSERT(nghttp2_hd_table_get(&deflater, i)->nv.value != NULL);
+  }
+
+  CU_ASSERT(0 == nghttp2_hd_change_table_size(&deflater, 156));
+  CU_ASSERT(4 == deflater.hd_table.len);
+  CU_ASSERT(4 == deflater.local_hd_tablelen);
+  CU_ASSERT(156 == deflater.local_hd_table_bufsize);
+
+  blocklen = nghttp2_hd_deflate_hd(&deflater, &buf, &buflen, 0, &nv3, 1);
+  CU_ASSERT(blocklen > 0);
+  /* Now header table should be empty */
+  CU_ASSERT(0 == deflater.hd_table.len);
+  CU_ASSERT(0 == deflater.local_hd_tablelen);
+  CU_ASSERT(0 == deflater.local_hd_table_bufsize);
+
+  nghttp2_hd_deflate_free(&deflater);
+
+  /* Check more complex use case */
+  nghttp2_hd_deflate_init2(&deflater, NGHTTP2_HD_SIDE_REQUEST,
+                           155);
+  nghttp2_hd_inflate_init(&inflater, NGHTTP2_HD_SIDE_REQUEST);
+  blocklen = nghttp2_hd_deflate_hd(&deflater, &buf, &buflen, 0,
+                                   nva1, ARRLEN(nva1));
+  CU_ASSERT(blocklen > 0);
+  /* Now header table should look like this:
+   *
+   *  0: k1000, v100 (R)
+   *  1: k100, v100 (R)
+   *  2: k10, v10 (R)
+   *  3: k1, v1 (-)  <- name, value must be NULL and not in reference set
+   *
+   * But due to the local table size limit, name/value of index=3 must
+   * be NULL.
+   */
+  CU_ASSERT(4 == deflater.hd_table.len);
+  CU_ASSERT(3 == deflater.local_hd_tablelen);
+  CU_ASSERT(120 == deflater.local_hd_table_bufsize);
+  for(i = 0; i < 3; ++i) {
+    CU_ASSERT(nghttp2_hd_table_get(&deflater, i)->nv.name != NULL);
+    CU_ASSERT(nghttp2_hd_table_get(&deflater, i)->nv.value != NULL);
+  }
+  ent = nghttp2_hd_table_get(&deflater, 3);
+  CU_ASSERT(ent->nv.name == NULL);
+  CU_ASSERT(ent->nv.value == NULL);
+  CU_ASSERT(0 == (ent->flags & NGHTTP2_HD_FLAG_REFSET));
+
+  rv = nghttp2_hd_inflate_hd(&inflater, &resnva, buf, blocklen);
+  CU_ASSERT(4 == rv);
+  assert_nv_equal(nva1, resnva, 4);
+  nghttp2_hd_end_headers(&inflater);
+  nghttp2_nv_array_del(resnva);
+
+  blocklen = nghttp2_hd_deflate_hd(&deflater, &buf, &buflen, 0,
+                                   nva2, ARRLEN(nva2));
+  CU_ASSERT(blocklen > 0);
+  /* Now header table should look like this:
+   *
+   *  0: k1, v1 (R)
+   *  1: k1000, v100 (R)
+   *  2: k100, v100 (R)
+   *  3: k10, v10 (-) <- name, value must be NULL
+   *  4: k1, v1   (-) <- name, value must be NULL
+   */
+  CU_ASSERT(5 == deflater.hd_table.len);
+  CU_ASSERT(3 == deflater.local_hd_tablelen);
+  CU_ASSERT(118 == deflater.local_hd_table_bufsize);
+  ent = nghttp2_hd_table_get(&deflater, 3);
+  CU_ASSERT(0 == (ent->flags & NGHTTP2_HD_FLAG_REFSET));
+  ent = nghttp2_hd_table_get(&deflater, 3);
+  CU_ASSERT(0 == (ent->flags & NGHTTP2_HD_FLAG_REFSET));
+
+  rv = nghttp2_hd_inflate_hd(&inflater, &resnva, buf, blocklen);
+  CU_ASSERT(2 == rv);
+  /* Sort before comparison */
+  nghttp2_nv_array_sort(nva2, 2);
+  assert_nv_equal(nva2, resnva, 2);
+  nghttp2_hd_end_headers(&inflater);
+  nghttp2_nv_array_del(resnva);
+
+  blocklen = nghttp2_hd_deflate_hd(&deflater, &buf, &buflen, 0, &nv3, 1);
+  CU_ASSERT(blocklen > 0);
+  /* Now header table should look like this:
+   *
+   *  0: a..a, a..a (-)
+   *  1: k1, v1 (-)
+   *  2: k1000, v100 (-)
+   *  3: k100, v100 (-)
+   *  4: k10, v10 (-)
+   *  5: k1, v1   (-)
+   *
+   *  name/value of all entries must be NULL.
+   */
+  CU_ASSERT(6 == deflater.hd_table.len);
+  CU_ASSERT(0 == deflater.local_hd_tablelen);
+  CU_ASSERT(0 == deflater.local_hd_table_bufsize);
+  for(i = 0; i < 6; ++i) {
+    ent = nghttp2_hd_table_get(&deflater, i);
+    CU_ASSERT(0 == (ent->flags & NGHTTP2_HD_FLAG_REFSET));
+  }
+
+  rv = nghttp2_hd_inflate_hd(&inflater, &resnva, buf, blocklen);
+  CU_ASSERT(1 == rv);
+  assert_nv_equal(&nv3, resnva, 1);
+  nghttp2_hd_end_headers(&inflater);
+  nghttp2_nv_array_del(resnva);
+
+  free(buf);
+  nghttp2_hd_inflate_free(&inflater);
+  nghttp2_hd_deflate_free(&deflater);
+
+}
+
 void test_nghttp2_hd_inflate_indname_inc(void)
 {
   nghttp2_hd_context inflater;
@@ -389,6 +580,40 @@ void test_nghttp2_hd_inflate_clearall_inc(void)
   nghttp2_hd_inflate_free(&inflater);
 }
 
+void test_nghttp2_hd_change_table_size(void)
+{
+  nghttp2_hd_context deflater;
+  nghttp2_nv nva[] = { MAKE_NV(":method", "GET"),
+                       MAKE_NV(":path", "/") };
+  uint8_t *buf = NULL;
+  size_t buflen = 0;
+  ssize_t rv;
+
+  nghttp2_hd_deflate_init(&deflater, NGHTTP2_HD_SIDE_REQUEST);
+  CU_ASSERT(0 == nghttp2_hd_change_table_size(&deflater, 8000));
+  CU_ASSERT(255 == deflater.hd_table.mask);
+  CU_ASSERT(8000 == deflater.hd_table_bufsize_max);
+
+  rv = nghttp2_hd_deflate_hd(&deflater, &buf, &buflen, 0, nva, 2);
+  CU_ASSERT(rv > 0);
+  CU_ASSERT(2 == deflater.hd_table.len);
+
+  CU_ASSERT(0 == nghttp2_hd_change_table_size(&deflater, 16384));
+  CU_ASSERT(511 == deflater.hd_table.mask);
+  CU_ASSERT(2 == deflater.hd_table.len);
+  CU_ASSERT(2 == deflater.local_hd_tablelen);
+  CU_ASSERT(5 ==
+            deflater.hd_table.buffer[deflater.hd_table.first]->nv.namelen);
+
+  CU_ASSERT(0 == nghttp2_hd_change_table_size(&deflater, 0));
+  CU_ASSERT(511 == deflater.hd_table.mask);
+  CU_ASSERT(0 == deflater.hd_table.len);
+  CU_ASSERT(0 == deflater.local_hd_tablelen);
+
+  free(buf);
+  nghttp2_hd_deflate_free(&deflater);
+}
+
 static void check_deflate_inflate(nghttp2_hd_context *deflater,
                                   nghttp2_hd_context *inflater,
                                   nghttp2_nv *nva, size_t nvlen)

tests/nghttp2_hd_test.h

@@ -28,10 +28,12 @@
 void test_nghttp2_hd_deflate(void);
 void test_nghttp2_hd_deflate_same_indexed_repr(void);
 void test_nghttp2_hd_deflate_common_header_eviction(void);
+void test_nghttp2_hd_deflate_local_buffer(void);
 void test_nghttp2_hd_inflate_indname_inc(void);
 void test_nghttp2_hd_inflate_indname_inc_eviction(void);
 void test_nghttp2_hd_inflate_newname_inc(void);
 void test_nghttp2_hd_inflate_clearall_inc(void);
+void test_nghttp2_hd_change_table_size(void);
 void test_nghttp2_hd_deflate_inflate(void);
 
 #endif /* NGHTTP2_HD_TEST_H */