Implement static Huffman for header compression

The current implementation uses Huffman code tables described
in http://tools.ietf.org/html/draft-rpeon-httpbis-header-compression-03

lib/Makefile.am

@@ -37,7 +37,8 @@ OBJECTS = nghttp2_pq.c nghttp2_map.c nghttp2_queue.c \
 	nghttp2_session.c nghttp2_submit.c \
 	nghttp2_helper.c \
 	nghttp2_npn.c nghttp2_gzip.c \
-	nghttp2_hd.c nghttp2_version.c
+	nghttp2_hd.c nghttp2_hd_huffman.c nghttp2_hd_huffman_data.c \
+	nghttp2_version.c
 
 HFILES = nghttp2_pq.h nghttp2_int.h nghttp2_map.h nghttp2_queue.h \
 	nghttp2_buffer.h nghttp2_frame.h \
@@ -45,7 +46,7 @@ HFILES = nghttp2_pq.h nghttp2_int.h nghttp2_map.h nghttp2_queue.h \
 	nghttp2_npn.h nghttp2_gzip.h \
 	nghttp2_submit.h nghttp2_outbound_item.h \
 	nghttp2_net.h \
-	nghttp2_hd.h
+	nghttp2_hd.h nghttp2_hd_huffman.h
 
 libnghttp2_la_SOURCES = $(HFILES) $(OBJECTS)
 libnghttp2_la_LDFLAGS = -no-undefined \

lib/nghttp2_hd.h

@@ -32,6 +32,7 @@
 #include <nghttp2/nghttp2.h>
 
 #define NGHTTP2_INITIAL_EMIT_SET_SIZE 128
+#define NGHTTP2_INITIAL_BUF_TRACK_SIZE 128
 
 #define NGHTTP2_HD_DEFAULT_MAX_BUFFER_SIZE (1 << 12)
 #define NGHTTP2_HD_MAX_ENTRY_SIZE 3072
@@ -58,7 +59,13 @@ typedef enum {
   /* Indicates that the entry is emitted in the current header
      processing. */
   NGHTTP2_HD_FLAG_EMIT = 1 << 3,
-  NGHTTP2_HD_FLAG_IMPLICIT_EMIT = 1 << 4
+  NGHTTP2_HD_FLAG_IMPLICIT_EMIT = 1 << 4,
+  /* Indicates that the name was gifted to the entry and no copying
+     necessary. */
+  NGHTTP2_HD_FLAG_NAME_GIFT = 1 << 5,
+  /* Indicates that the value was gifted to the entry and no copying
+     necessary. */
+  NGHTTP2_HD_FLAG_VALUE_GIFT = 1 << 6
 } nghttp2_hd_flags;
 
 typedef struct {
@@ -96,8 +103,18 @@ typedef struct {
   uint8_t bad;
   /* Role of this context; deflate or infalte */
   nghttp2_hd_role role;
+  /* Huffman compression side: NGHTTP2_HD_SIDE_CLIENT uses huffman
+     table for request. NGHTTP2_HD_SIDE_SERVER uses huffman table for
+     response. */
+  nghttp2_hd_side side;
   /* Maximum header table size */
   size_t hd_table_bufsize_max;
+  /* Keep track of allocated buffers in inflation */
+  uint8_t **buf_track;
+  /* The capacity of |buf_track| */
+  uint16_t buf_track_capacity;
+  /* The number of entry the |buf_track| contains. */
+  size_t buf_tracklen;
 } nghttp2_hd_context;
 
 /*
@@ -227,12 +244,14 @@ int nghttp2_hd_end_headers(nghttp2_hd_context *deflater_or_inflater);
 int nghttp2_hd_emit_indname_block(uint8_t **buf_ptr, size_t *buflen_ptr,
                                   size_t *offset_ptr, size_t index,
                                   const uint8_t *value, size_t valuelen,
-                                  int inc_indexing);
+                                  int inc_indexing,
+                                  nghttp2_hd_side side);
 
 /* For unittesting purpose */
 int nghttp2_hd_emit_newname_block(uint8_t **buf_ptr, size_t *buflen_ptr,
                                   size_t *offset_ptr, nghttp2_nv *nv,
-                                  int inc_indexing);
+                                  int inc_indexing,
+                                  nghttp2_hd_side side);
 
 /* For unittesting purpose */
 int nghttp2_hd_emit_subst_indname_block(uint8_t **buf_ptr, size_t *buflen_ptr,
@@ -249,4 +268,68 @@ int nghttp2_hd_emit_subst_newname_block(uint8_t **buf_ptr, size_t *buflen_ptr,
 nghttp2_hd_entry* nghttp2_hd_table_get(nghttp2_hd_context *context,
                                        size_t index);
 
+/* Huffman encoding/decoding functions */
+
+/*
+ * Counts the required bytes to encode |src| with length |len|. If
+ * |side| is NGHTTP2_HD_SIDE_CLIENT, the request huffman code table is
+ * used. Otherwise, the response code table is used.
+ *
+ * This function returns the number of required bytes to encode given
+ * data, including terminal symbol code. This function always
+ * succeeds.
+ */
+size_t nghttp2_hd_huff_encode_count(const uint8_t *src, size_t len,
+                                    nghttp2_hd_side side);
+
+/*
+ * Encodes the given data |src| with length |srclen| to the given
+ * memory location pointed by |dest|, allocated at lest |destlen|
+ * bytes. The caller is responsible to specify |destlen| at least the
+ * length that nghttp2_hd_huff_encode_count() returns.  If |side| is
+ * NGHTTP2_HD_SIDE_CLIENT, the request huffman code table is
+ * used. Otherwise, the response code table is used.
+ *
+ * This function returns the number of written bytes, including
+ * terminal symbol code. This return value is exactly the same with
+ * the return value of nghttp2_hd_huff_encode_count() if it is given
+ * with the same |src|, |srclen|, and |side|. This function always
+ * succeeds.
+ */
+ssize_t nghttp2_hd_huff_encode(uint8_t *dest, size_t destlen,
+                               const uint8_t *src, size_t srclen,
+                               nghttp2_hd_side side);
+
+/*
+ * Counts the number of required bytes to decode |src| with length
+ * |srclen|. The given input must be terminated with terminal code. If
+ * |side| is NGHTTP2_HD_SIDE_CLIENT, the request huffman code table is
+ * used. Otherwise, the response code table is used.
+ *
+ * This function returns the number of required bytes to decode given
+ * data if it succeeds, or -1.
+ */
+ssize_t nghttp2_hd_huff_decode_count(const uint8_t *src, size_t srclen,
+                                     nghttp2_hd_side side);
+
+/*
+ * Decodes the given data |src| with length |srclen| to the given
+ * memory location pointed by |dest|, allocated at lest |destlen|
+ * bytes. The given input must be terminated with terminal code. The
+ * caller is responsible to specify |destlen| at least the length that
+ * nghttp2_hd_huff_decode_count() returns.  If |side| is
+ * NGHTTP2_HD_SIDE_CLIENT, the request huffman code table is
+ * used. Otherwise, the response code table is used.
+ *
+ * This function returns the number of written bytes.  This return
+ * value is exactly the same with the return value of
+ * nghttp2_hd_huff_decode_count() if it is given with the same |src|,
+ * |srclen|, and |side|.
+ *
+ * This function returns -1 if it fails.
+ */
+ssize_t nghttp2_hd_huff_decode(uint8_t *dest, size_t destlen,
+                               const uint8_t *src, size_t srclen,
+                               nghttp2_hd_side side);
+
 #endif /* NGHTTP2_HD_COMP_H */

lib/nghttp2_hd_huffman.c

+/*
+ * nghttp2 - HTTP/2.0 C Library
+ *
+ * Copyright (c) 2013 Tatsuhiro Tsujikawa
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "nghttp2_hd_huffman.h"
+
+#include <string.h>
+#include <assert.h>
+
+#include "nghttp2_hd.h"
+
+extern const nghttp2_huff_sym req_huff_sym_table[];
+extern const int16_t req_huff_decode_table[][256];
+
+extern const nghttp2_huff_sym res_huff_sym_table[];
+extern const int16_t res_huff_decode_table[][256];
+
+/*
+ * Returns next 8 bits of data from |in|, starting |bitoff| bits
+ * offset. If there are fewer bits left than |bitoff|, the left bits
+ * with padded with 0 are returned. The |bitoff| must be strictly less
+ * than 8.
+ */
+static uint8_t get_prefix_byte(const uint8_t *in, size_t len, size_t bitoff)
+{
+  uint8_t b;
+  size_t bitleft;
+  if(bitoff == 0) {
+    return *in;
+  }
+  bitleft = 8 - bitoff;
+  b = (*in & ((1 << bitleft) - 1)) << bitoff;
+  if(len > 1) {
+    b |= *(in + 1) >> bitleft;
+  }
+  return b;
+}
+
+/*
+ * Decodes next byte from input |in| with length |len|, starting
+ * |bitoff| bit offset.
+ *
+ * This function returns the decoded symbol number (0-255 and 256 for
+ * special terminal symbol) if it succeeds, or -1.
+ */
+static int huff_decode(const uint8_t *in, size_t len, size_t bitoff,
+                       const nghttp2_huff_sym *huff_sym_table,
+                       const huff_decode_table_type *huff_decode_table)
+{
+  int rv = 0;
+  size_t len_orig = len;
+  if(len == 0) {
+    return -1;
+  }
+  for(;;) {
+    rv = huff_decode_table[rv][get_prefix_byte(in, len, bitoff)];
+    if(rv >= 0) {
+      break;
+    }
+    /* Negative return value means we need to lookup next table. */
+    rv = -rv;
+    ++in;
+    --len;
+    if(len == 0) {
+      return -1;
+    }
+  }
+  if(bitoff + huff_sym_table[rv].nbits > len_orig * 8) {
+    return -1;
+  }
+  return rv;
+}
+
+/*
+ * Returns next LSB aligned |nbits| bits from huffman symbol |sym|,
+ * starting |codebitoff| bit offset (from beginning of code sequence,
+ * so it could be more than 8).
+ */
+static uint8_t huff_get_lsb_aligned(const nghttp2_huff_sym *sym,
+                                    size_t codebitoff,
+                                    size_t nbits)
+{
+  uint8_t a = sym->code[codebitoff/8];
+  size_t localbitoff = codebitoff & 0x7;
+  size_t bitleft = 8 - localbitoff;
+
+  if(bitleft >= nbits) {
+    return (a >> (bitleft - nbits)) & ((1 << nbits) - 1);
+  } else {
+    uint8_t b = 0;
+    a &= ((1 << bitleft) - 1);
+    a <<= nbits - bitleft;
+    if((sym->nbits + 7) / 8 > codebitoff / 8 + 1) {
+      b = sym->code[codebitoff / 8 + 1] >> (8 - (nbits - bitleft));
+    }
+    return a | b;
+  }
+}
+
+/*
+ * Encodes huffman code |sym| into |*dest_ptr|,starting |bitoff|
+ * offset. The |bitoff| must be strictly less than 8. At the end of
+ * the process, the |*dest_ptr| is updated and points where next
+ * output should be placed. The bit offset of the pointed location is
+ * returned.
+ */
+static size_t huff_encode_sym(uint8_t **dest_ptr, size_t bitoff,
+                              const nghttp2_huff_sym *sym)
+{
+  size_t b = 0;
+  **dest_ptr |= huff_get_lsb_aligned(sym, b, 8 - bitoff);
+  b += 8 - bitoff;
+  ++*dest_ptr;
+  for(; b < sym->nbits; b += 8, ++*dest_ptr) {
+    **dest_ptr |= huff_get_lsb_aligned(sym, b, 8);
+  }
+  bitoff = 8 - (b - sym->nbits);
+  if(bitoff > 0) {
+    --*dest_ptr;
+  }
+  return bitoff;
+}
+
+size_t nghttp2_hd_huff_encode_count(const uint8_t *src, size_t len,
+                                    nghttp2_hd_side side)
+{
+  size_t i;
+  size_t nbits = 0;
+  const nghttp2_huff_sym *huff_sym_table;
+
+  if(side == NGHTTP2_HD_SIDE_CLIENT) {
+    huff_sym_table = req_huff_sym_table;
+  } else {
+    huff_sym_table = res_huff_sym_table;
+  }
+  for(i = 0; i < len; ++i) {
+    nbits += huff_sym_table[src[i]].nbits;
+  }
+  /* 256 is special terminal symbol */
+  return (nbits + huff_sym_table[256].nbits + 7) / 8;
+}
+
+ssize_t nghttp2_hd_huff_encode(uint8_t *dest, size_t destlen,
+                               const uint8_t *src, size_t srclen,
+                               nghttp2_hd_side side)
+{
+  int bitoff = 0;
+  uint8_t *dest_first = dest;
+  size_t i;
+  const nghttp2_huff_sym *huff_sym_table;
+
+  if(side == NGHTTP2_HD_SIDE_CLIENT) {
+    huff_sym_table = req_huff_sym_table;
+  } else {
+    huff_sym_table = res_huff_sym_table;
+  }
+  memset(dest, 0, destlen);
+  for(i = 0; i < srclen; ++i) {
+    const nghttp2_huff_sym *sym = &huff_sym_table[src[i]];
+    bitoff = huff_encode_sym(&dest, bitoff, sym);
+  }
+  /* 256 is special terminal symbol */
+  bitoff = huff_encode_sym(&dest, bitoff, &huff_sym_table[256]);
+  return dest - dest_first + (bitoff > 0);
+}
+
+ssize_t nghttp2_hd_huff_decode_count(const uint8_t *src, size_t srclen,
+                                     nghttp2_hd_side side)
+{
+  size_t bitoff = 0;
+  size_t i, j;
+  const nghttp2_huff_sym *huff_sym_table;
+  const huff_decode_table_type *huff_decode_table;
+
+  if(side == NGHTTP2_HD_SIDE_CLIENT) {
+    huff_sym_table = req_huff_sym_table;
+    huff_decode_table = req_huff_decode_table;
+  } else {
+    huff_sym_table = res_huff_sym_table;
+    huff_decode_table = res_huff_decode_table;
+  }
+  j = 0;
+  for(i = 0; i < srclen;) {
+    int rv = huff_decode(src + i, srclen - i, bitoff,
+                         huff_sym_table, huff_decode_table);
+    if(rv == -1) {
+      return -1;
+    }
+    if(rv == 256) {
+      /* 256 is special terminal symbol */
+      break;
+    }
+    j++;
+    bitoff += huff_sym_table[rv].nbits;
+    i += bitoff / 8;
+    bitoff &= 0x7;
+  }
+  return j;
+}
+
+ssize_t nghttp2_hd_huff_decode(uint8_t *dest, size_t destlen,
+                               const uint8_t *src, size_t srclen,
+                               nghttp2_hd_side side)
+{
+  size_t bitoff = 0;
+  size_t i, j;
+  const nghttp2_huff_sym *huff_sym_table;
+  const huff_decode_table_type *huff_decode_table;
+
+  if(side == NGHTTP2_HD_SIDE_CLIENT) {
+    huff_sym_table = req_huff_sym_table;
+    huff_decode_table = req_huff_decode_table;
+  } else {
+    huff_sym_table = res_huff_sym_table;
+    huff_decode_table = res_huff_decode_table;
+  }
+  j = 0;
+  for(i = 0; i < srclen;) {
+    int rv = huff_decode(src + i, srclen - i, bitoff,
+                         huff_sym_table, huff_decode_table);
+    if(rv == -1) {
+      return -1;
+    }
+    if(rv == 256) {
+      /* 256 is special terminal symbol */
+      break;
+    }
+    dest[j++] = rv;
+    bitoff += huff_sym_table[rv].nbits;
+    i += bitoff / 8;
+    bitoff &= 0x7;
+  }
+  return j;
+}

lib/nghttp2_hd_huffman.h

+/*
+ * nghttp2 - HTTP/2.0 C Library
+ *
+ * Copyright (c) 2013 Tatsuhiro Tsujikawa
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+#ifndef NGHTTP2_HD_HUFFMAN_H
+#define NGHTTP2_HD_HUFFMAN_H
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif /* HAVE_CONFIG_H */
+
+#include <nghttp2/nghttp2.h>
+
+typedef int16_t huff_decode_table_type[256];
+
+typedef struct {
+  /* The number of bits in this code */
+  size_t nbits;
+  /* Code sequence padded with 0 */
+  uint8_t code[4];
+} nghttp2_huff_sym;
+
+#endif // NGHTTP2_HD_HUFFMAN_H

tests/nghttp2_hd_test.c

@@ -246,7 +246,8 @@ void test_nghttp2_hd_inflate_indname_inc(void)
   nghttp2_hd_inflate_init(&inflater, NGHTTP2_HD_SIDE_SERVER);
 
   CU_ASSERT(0 == nghttp2_hd_emit_indname_block(&buf, &buflen, &offset, 11,
-                                               nv.value, nv.valuelen, 1));
+                                               nv.value, nv.valuelen, 1,
+                                               NGHTTP2_HD_SIDE_CLIENT));
   CU_ASSERT(1 == nghttp2_hd_inflate_hd(&inflater, &resnva, buf, offset));
   assert_nv_equal(&nv, resnva, 1);
   CU_ASSERT(1 == inflater.hd_table.len);
@@ -270,13 +271,17 @@ void test_nghttp2_hd_inflate_indname_inc_eviction(void)
 
   memset(value, '0', sizeof(value));
   CU_ASSERT(0 == nghttp2_hd_emit_indname_block(&buf, &buflen, &offset, 2,
-                                               value, sizeof(value), 1));
+                                               value, sizeof(value), 1,
+                                               NGHTTP2_HD_SIDE_CLIENT));
   CU_ASSERT(0 == nghttp2_hd_emit_indname_block(&buf, &buflen, &offset, 3,
-                                               value, sizeof(value), 1));
+                                               value, sizeof(value), 1,
+                                               NGHTTP2_HD_SIDE_CLIENT));
   CU_ASSERT(0 == nghttp2_hd_emit_indname_block(&buf, &buflen, &offset, 4,
-                                               value, sizeof(value), 1));
+                                               value, sizeof(value), 1,
+                                               NGHTTP2_HD_SIDE_CLIENT));
   CU_ASSERT(0 == nghttp2_hd_emit_indname_block(&buf, &buflen, &offset, 5,
-                                               value, sizeof(value), 1));
+                                               value, sizeof(value), 1,
+                                               NGHTTP2_HD_SIDE_CLIENT));
 
   CU_ASSERT(4 == nghttp2_hd_inflate_hd(&inflater, &resnva, buf, offset));
   CU_ASSERT(5 == resnva[0].namelen);
@@ -304,7 +309,8 @@ void test_nghttp2_hd_inflate_newname_inc(void)
   nghttp2_hd_inflate_init(&inflater, NGHTTP2_HD_SIDE_SERVER);
 
   CU_ASSERT(0 == nghttp2_hd_emit_newname_block(&buf, &buflen, &offset,
-                                               &nv, 1));
+                                               &nv, 1,
+                                               NGHTTP2_HD_SIDE_CLIENT));
   CU_ASSERT(1 == nghttp2_hd_inflate_hd(&inflater, &resnva, buf, offset));
   assert_nv_equal(&nv, resnva, 1);
   CU_ASSERT(1 == inflater.hd_table.len);
@@ -336,7 +342,8 @@ void test_nghttp2_hd_inflate_clearall_inc(void)
   nghttp2_hd_inflate_init(&inflater, NGHTTP2_HD_SIDE_SERVER);
 
   CU_ASSERT(0 == nghttp2_hd_emit_newname_block(&buf, &buflen, &offset,
-                                               &nv, 1));
+                                               &nv, 1,
+                                               NGHTTP2_HD_SIDE_CLIENT));
   CU_ASSERT(1 == nghttp2_hd_inflate_hd(&inflater, &resnva, buf, offset));
   assert_nv_equal(&nv, resnva, 1);
   CU_ASSERT(0 == inflater.hd_table.len);
@@ -358,7 +365,8 @@ void test_nghttp2_hd_inflate_clearall_inc(void)
 
   offset = 0;
   CU_ASSERT(0 == nghttp2_hd_emit_newname_block(&buf, &buflen, &offset,
-                                               &nv, 1));
+                                               &nv, 1,
+                                               NGHTTP2_HD_SIDE_CLIENT));
   CU_ASSERT(1 == nghttp2_hd_inflate_hd(&inflater, &resnva, buf, offset));
   assert_nv_equal(&nv, resnva, 1);
   CU_ASSERT(1 == inflater.hd_table.len);