Rework outbound frame buffers

lib/nghttp2_buf.c

@@ -123,11 +123,17 @@ static void nghttp2_buf_chain_del(nghttp2_buf_chain *chain)
 
 int nghttp2_bufs_init(nghttp2_bufs *bufs, size_t chunk_length,
                       size_t max_chunk)
+{
+  return nghttp2_bufs_init2(bufs, chunk_length, max_chunk, 0);
+}
+
+int nghttp2_bufs_init2(nghttp2_bufs *bufs, size_t chunk_length,
+                       size_t max_chunk, size_t offset)
 {
   int rv;
   nghttp2_buf_chain *chain;
 
-  if(max_chunk == 0) {
+  if(max_chunk == 0 || chunk_length < offset) {
     return NGHTTP2_ERR_INVALID_ARGUMENT;
   }
 
@@ -136,9 +142,13 @@ int nghttp2_bufs_init(nghttp2_bufs *bufs, size_t chunk_length,
     return rv;
   }
 
+  bufs->offset = offset;
+
   bufs->head = chain;
   bufs->cur = bufs->head;
 
+  nghttp2_buf_shift_right(&bufs->cur->buf, offset);
+
   bufs->chunk_length = chunk_length;
   bufs->chunk_left = max_chunk - 1;
 
@@ -158,10 +168,36 @@ void nghttp2_bufs_free(nghttp2_bufs *bufs)
   }
 }
 
+void nghttp2_bufs_seek_last_present(nghttp2_bufs *bufs)
+{
+  nghttp2_buf_chain *ci;
+
+  for(ci = bufs->cur; ci->next; ci = ci->next) {
+    if(nghttp2_buf_len(&ci->buf) == 0) {
+      return;
+    } else {
+      bufs->cur = ci;
+    }
+  }
+}
+
+ssize_t nghttp2_bufs_len(nghttp2_bufs *bufs)
+{
+  nghttp2_buf_chain *ci;
+  ssize_t len;
+
+  len = 0;
+  for(ci = bufs->head; ci; ci = ci->next) {
+    len += nghttp2_buf_len(&ci->buf);
+  }
+
+  return len;
+}
+
 static int nghttp2_bufs_avail(nghttp2_bufs *bufs)
 {
   return nghttp2_buf_avail(&bufs->cur->buf) +
-    bufs->chunk_left * bufs->chunk_left;
+    (bufs->chunk_left - bufs->offset) * bufs->chunk_left;
 }
 
 static int nghttp2_bufs_alloc_chain(nghttp2_bufs *bufs)
@@ -189,6 +225,8 @@ static int nghttp2_bufs_alloc_chain(nghttp2_bufs *bufs)
   bufs->cur->next = chain;
   bufs->cur = chain;
 
+  nghttp2_buf_shift_right(&bufs->cur->buf, bufs->offset);
+
   return 0;
 }
 
@@ -199,7 +237,7 @@ int nghttp2_bufs_add(nghttp2_bufs *bufs, const void *data, size_t len)
   nghttp2_buf *buf;
   const uint8_t *p;
 
-  if((size_t)nghttp2_bufs_avail(bufs) < len) {
+  if(nghttp2_bufs_avail(bufs) < (ssize_t)len) {
     return NGHTTP2_ERR_BUFFER_ERROR;
   }
 
@@ -224,7 +262,7 @@ int nghttp2_bufs_add(nghttp2_bufs *bufs, const void *data, size_t len)
   return 0;
 }
 
-int nghttp2_bufs_addb(nghttp2_bufs *bufs, uint8_t b)
+static int nghttp2_bufs_ensure_addb(nghttp2_bufs *bufs)
 {
   int rv;
   nghttp2_buf *buf;
@@ -232,7 +270,6 @@ int nghttp2_bufs_addb(nghttp2_bufs *bufs, uint8_t b)
   buf = &bufs->cur->buf;
 
   if(nghttp2_buf_avail(buf) > 0) {
-    *buf->last++ = b;
     return 0;
   }
 
@@ -241,9 +278,61 @@ int nghttp2_bufs_addb(nghttp2_bufs *bufs, uint8_t b)
     return rv;
   }
 
-  buf = &bufs->cur->buf;
+  return 0;
+}
+
+int nghttp2_bufs_addb(nghttp2_bufs *bufs, uint8_t b)
+{
+  int rv;
+
+  rv = nghttp2_bufs_ensure_addb(bufs);
+  if(rv != 0) {
+    return rv;
+  }
+
+  *bufs->cur->buf.last++ = b;
+
+  return 0;
+}
+
+int nghttp2_bufs_addb_hold(nghttp2_bufs *bufs, uint8_t b)
+{
+  int rv;
 
-  *buf->last++ = b;
+  rv = nghttp2_bufs_ensure_addb(bufs);
+  if(rv != 0) {
+    return rv;
+  }
+
+  *bufs->cur->buf.last = b;
+
+  return 0;
+}
+
+int nghttp2_bufs_orb(nghttp2_bufs *bufs, uint8_t b)
+{
+  int rv;
+
+  rv = nghttp2_bufs_ensure_addb(bufs);
+  if(rv != 0) {
+    return rv;
+  }
+
+  *bufs->cur->buf.last++ |= b;
+
+  return 0;
+}
+
+int nghttp2_bufs_orb_hold(nghttp2_bufs *bufs, uint8_t b)
+{
+  int rv;
+
+  rv = nghttp2_bufs_ensure_addb(bufs);
+  if(rv != 0) {
+    return rv;
+  }
+
+  *bufs->cur->buf.last |= b;
 
   return 0;
 }
@@ -272,7 +361,9 @@ ssize_t nghttp2_bufs_remove(nghttp2_bufs *bufs, uint8_t **out)
   for(chain = bufs->head; chain; chain = chain->next) {
     buf = &chain->buf;
     resbuf.last = nghttp2_cpymem(resbuf.last, buf->pos, nghttp2_buf_len(buf));
+
     nghttp2_buf_reset(buf);
+    nghttp2_buf_shift_right(&chain->buf, bufs->offset);
   }
 
   bufs->cur = bufs->head;
@@ -288,5 +379,23 @@ void nghttp2_bufs_reset(nghttp2_bufs *bufs)
 
   for(chain = bufs->head; chain; chain = chain->next) {
     nghttp2_buf_reset(&chain->buf);
+    nghttp2_buf_shift_right(&chain->buf, bufs->offset);
   }
+
+  bufs->cur = bufs->head;
 }
+
+int nghttp2_bufs_advance(nghttp2_bufs *bufs)
+{
+  return nghttp2_bufs_alloc_chain(bufs);
+}
+
+int nghttp2_bufs_next_present(nghttp2_bufs *bufs)
+{
+  nghttp2_buf_chain *chain;
+
+  chain = bufs->cur->next;
+
+  return chain && nghttp2_buf_len(&chain->buf);
+}
+

lib/nghttp2_buf.h

@@ -134,39 +134,202 @@ void nghttp2_buf_reset(nghttp2_buf *buf);
  */
 void nghttp2_buf_wrap_init(nghttp2_buf *buf, uint8_t *begin, size_t len);
 
-/*
- * List of nghttp2_buf
- */
-
 struct nghttp2_buf_chain;
 
 typedef struct nghttp2_buf_chain nghttp2_buf_chain;
 
+/* Chains 2 buffers */
 struct nghttp2_buf_chain {
+  /* Points to the subsequent buffer. NULL if there is no such
+     buffer. */
   nghttp2_buf_chain *next;
   nghttp2_buf buf;
 };
 
 typedef struct {
+  /* Points to the first buffer */
   nghttp2_buf_chain *head;
+  /* Buffer pointer where write occurs. */
   nghttp2_buf_chain *cur;
   /* The buffer capacity of each buf */
   size_t chunk_length;
   /* The maximum number of nghttp2_buf_chain */
   size_t chunk_left;
+  /* pos offset from begin in each buffers. On initialization and
+     reset, buf->pos and buf->last are positioned at buf->begin +
+     offset. */
+  size_t offset;
 } nghttp2_bufs;
 
+/*
+ * This is the same as calling nghttp2_bufs_init2 with the given
+ * arguments and offset = 0.
+ */
 int nghttp2_bufs_init(nghttp2_bufs *bufs, size_t chunk_length,
                       size_t max_chunk);
 
+/*
+ * Initializes |bufs|. Each buffer size is given in the
+ * |chunk_length|.  The maximum number of buffers is given in the
+ * |max_chunk|. Each buffer will have bufs->pos and bufs->last shifted
+ * to left by |offset| bytes on creation and reset.
+ *
+ * This function allocates first buffer.  bufs->head and bufs->cur
+ * will point to the first buffer after this call.
+ *
+ * This function returns 0 if it succeeds, or one of the following
+ * negative error codes:
+ *
+ * NGHTTP2_ERR_NOMEM
+ *     Out of memory.
+ * NGHTTP2_ERR_INVALID_ARGUMENT
+ *     max_chunk is 0
+ */
+int nghttp2_bufs_init2(nghttp2_bufs *bufs, size_t chunk_length,
+                       size_t max_chunk, size_t offset);
+
+/*
+ * Frees any related resources to the |bufs|.
+ */
 void nghttp2_bufs_free(nghttp2_bufs *bufs);
 
+/*
+ * Appends the |data| of length |len| to the |bufs|. The write starts
+ * at bufs->cur->buf.last. A new buffers will be allocated to store
+ * all data.
+ *
+ * This function returns 0 if it succeeds, or one of the following
+ * negative error codes:
+ *
+ * NGHTTP2_ERR_NOMEM
+ *     Out of memory.
+ * NGHTTP2_ERR_BUFFER_ERROR
+ *     Out of buffer space.
+ */
 int nghttp2_bufs_add(nghttp2_bufs *bufs, const void *data, size_t len);
 
+/*
+ * Appends a single byte |b| to the |bufs|. The write starts at
+ * bufs->cur->buf.last. A new buffers will be allocated to store all
+ * data.
+ *
+ * This function returns 0 if it succeeds, or one of the following
+ * negative error codes:
+ *
+ * NGHTTP2_ERR_NOMEM
+ *     Out of memory.
+ * NGHTTP2_ERR_BUFFER_ERROR
+ *     Out of buffer space.
+ */
 int nghttp2_bufs_addb(nghttp2_bufs *bufs, uint8_t b);
 
+/*
+ * Behaves like nghttp2_bufs_addb(), but this does not update
+ * buf->last pointer.
+ */
+int nghttp2_bufs_addb_hold(nghttp2_bufs *bufs, uint8_t b);
+
+#define nghttp2_bufs_fast_addb(BUFS, B)          \
+  do {                                           \
+    *(BUFS)->cur->buf.last++ = B;                \
+  } while(0)
+
+#define nghttp2_bufs_fast_addb_hold(BUFS, B)     \
+  do {                                           \
+    *(BUFS)->cur->buf.last = B;                  \
+  } while(0)
+
+/*
+ * Performs bitwise-OR of |b| at bufs->cur->buf.last. A new buffers
+ * will be allocated if necessary.
+ *
+ * This function returns 0 if it succeeds, or one of the following
+ * negative error codes:
+ *
+ * NGHTTP2_ERR_NOMEM
+ *     Out of memory.
+ * NGHTTP2_ERR_BUFFER_ERROR
+ *     Out of buffer space.
+ */
+int nghttp2_bufs_orb(nghttp2_bufs *bufs, uint8_t b);
+
+/*
+ * Behaves like nghttp2_bufs_orb(), but does not update buf->last
+ * pointer.
+ */
+int nghttp2_bufs_orb_hold(nghttp2_bufs *bufs, uint8_t b);
+
+#define nghttp2_bufs_fast_orb(BUFS, B)          \
+  do {                                          \
+    *(BUFS)->cur->buf.last++ |= B;              \
+  } while(0)
+
+#define nghttp2_bufs_fast_orb_hold(BUFS, B)     \
+  do {                                          \
+    *(BUFS)->cur->buf.last |= B;                \
+  } while(0)
+
+/*
+ * Copies all data stored in |bufs| to the contagious buffer.  This
+ * function allocates the contagious memory to store all data in
+ * |bufs| and assigns it to |*out|.
+ *
+ * On successful return, nghttp2_bufs_len(bufs) returns 0, just like
+ * after calling nghttp2_bufs_reset().
+
+ * This function returns the length of copied data and assigns the
+ * pointer to copied data to |*out| if it succeeds, or one of the
+ * following negative error codes:
+ *
+ * NGHTTP2_ERR_NOMEM
+ *     Out of memory
+ */
 ssize_t nghttp2_bufs_remove(nghttp2_bufs *bufs, uint8_t **out);
 
+/*
+ * Resets |bufs| and makes the buffers empty.
+ */
 void nghttp2_bufs_reset(nghttp2_bufs *bufs);
 
+/*
+ * Moves bufs->cur to bufs->cur->next.  If resulting bufs->cur is
+ * NULL, this function allocates new buffers and bufs->cur points to
+ * it.
+ *
+ * This function returns 0 if it succeeds, or one of the following
+ * negative error codes:
+ *
+ * NGHTTP2_ERR_NOMEM
+ *     Out of memory
+ * NGHTTP2_ERR_BUFFER_ERROR
+ *     Out of buffer space.
+ */
+int nghttp2_bufs_advance(nghttp2_bufs *bufs);
+
+/* Sets bufs->cur to bufs->head */
+#define nghttp2_bufs_rewind(BUFS)               \
+  do {                                          \
+    (BUFS)->cur = (BUFS)->head;                 \
+  } while(0)
+
+/*
+ * Move bufs->cur, from the current position, using next member, to
+ * the last buf which has nghttp2_buf_len(buf) > 0 without seeing buf
+ * which satisfies nghttp2_buf_len(buf) == 0. If bufs->cur->next is
+ * NULL, bufs->cur is unchanged.
+ */
+void nghttp2_bufs_seek_last_present(nghttp2_bufs *bufs);
+
+/*
+ * Returns nonzero if bufs->cur->next is not emtpy.
+ */
+int nghttp2_bufs_next_present(nghttp2_bufs *bufs);
+
+#define nghttp2_bufs_cur_avail(BUFS) nghttp2_buf_avail(&(BUFS)->cur->buf)
+
+/*
+ * Returns the buffer length of |bufs|.
+ */
+ssize_t nghttp2_bufs_len(nghttp2_bufs *bufs);
+
 #endif /* NGHTTP2_BUF_H */

lib/nghttp2_hd.h

@@ -295,28 +295,28 @@ int nghttp2_hd_inflate_change_table_size(nghttp2_hd_inflater *inflater,
 
 /*
  * Deflates the |nva|, which has the |nvlen| name/value pairs, into
- * the buffer pointed by the |buf|. The caller must ensure that
- * nghttp2_buf_len(buf) == 0 holds. Write starts at buf->last.
+ * the |bufs|.
  *
- * This function expands |buf| as necessary to store the result.
+ * This function expands |bufs| as necessary to store the result. If
+ * buffers is full and the process still requires more space, this
+ * funtion fails and returns NGHTTP2_ERR_HEADER_COMP.
  *
- * This function copies necessary data into |buf|. After this function
- * returns, it is safe to delete the |nva|.
+ * After this function returns, it is safe to delete the |nva|.
  *
  * TODO: The rest of the code call nghttp2_hd_end_headers() after this
  * call, but it is just a regacy of the first implementation. Now it
  * is not required to be called as of now.
  *
- * This function returns the number of bytes outputted if it succeeds,
- * or one of the following negative error codes:
+ * This function returns 0 if it succeeds, or one of the following
+ * negative error codes:
  *
  * NGHTTP2_ERR_NOMEM
  *     Out of memory.
  * NGHTTP2_ERR_HEADER_COMP
  *     Deflation process has failed.
  */
-ssize_t nghttp2_hd_deflate_hd(nghttp2_hd_deflater *deflater,
-                              nghttp2_buf *buf,
+int nghttp2_hd_deflate_hd(nghttp2_hd_deflater *deflater,
+                          nghttp2_bufs *bufs,
                           nghttp2_nv *nva, size_t nvlen);
 
 typedef enum {
@@ -369,16 +369,16 @@ ssize_t nghttp2_hd_inflate_hd(nghttp2_hd_inflater *inflater,
 int nghttp2_hd_inflate_end_headers(nghttp2_hd_inflater *inflater);
 
 /* For unittesting purpose */
-int nghttp2_hd_emit_indname_block(nghttp2_buf *buf, size_t index,
+int nghttp2_hd_emit_indname_block(nghttp2_bufs *bufs, size_t index,
                                   const uint8_t *value, size_t valuelen,
                                   int inc_indexing);
 
 /* For unittesting purpose */
-int nghttp2_hd_emit_newname_block(nghttp2_buf *buf, nghttp2_nv *nv,
+int nghttp2_hd_emit_newname_block(nghttp2_bufs *bufs, nghttp2_nv *nv,
                                   int inc_indexing);
 
 /* For unittesting purpose */
-int nghttp2_hd_emit_table_size(nghttp2_buf *buf, size_t table_size);
+int nghttp2_hd_emit_table_size(nghttp2_bufs *bufs, size_t table_size);
 
 /* For unittesting purpose */
 nghttp2_hd_entry* nghttp2_hd_table_get(nghttp2_hd_context *context,
@@ -396,18 +396,18 @@ nghttp2_hd_entry* nghttp2_hd_table_get(nghttp2_hd_context *context,
 size_t nghttp2_hd_huff_encode_count(const uint8_t *src, size_t len);
 
 /*
- * Encodes the given data |src| with length |srclen| to the given
- * memory location pointed by |dest|, allocated at lest |destlen|
- * bytes. The caller is responsible to specify |destlen| at least the
- * length that nghttp2_hd_huff_encode_count() returns.
- *
- * This function returns the number of written bytes, including
- * padding of prefix of terminal symbol code. This return value is
- * exactly the same with the return value of
- * nghttp2_hd_huff_encode_count() if it is given with the same |src|
- * and |srclen|. This function always succeeds.
+ * Encodes the given data |src| with length |srclen| to the |bufs|.
+ * This function expands extra buffers in |bufs| if necessary.
+ *
+ * This function returns 0 if it succeeds, or one of the following
+ * negative error codes:
+ *
+ * NGHTTP2_ERR_NOMEM
+ *     Out of memory.
+ * NGHTTP2_ERR_BUFFER_ERROR
+ *     Out of buffer space.
  */
-ssize_t nghttp2_hd_huff_encode(uint8_t *dest, size_t destlen,
+int nghttp2_hd_huff_encode(nghttp2_bufs *bufs,
                            const uint8_t *src, size_t srclen);
 
 void nghttp2_hd_huff_decode_context_init(nghttp2_hd_huff_decode_context *ctx);

lib/nghttp2_hd_huffman.c

@@ -40,24 +40,60 @@ extern const nghttp2_huff_decode huff_decode_table[][16];
  * and points where next output should be placed. The number of
  * unfilled bits in the pointed location is returned.
  */
-static size_t huff_encode_sym(uint8_t **dest_ptr, size_t rembits,
+static ssize_t huff_encode_sym(nghttp2_bufs *bufs, size_t *avail_ptr,
+                               size_t rembits,
                                const nghttp2_huff_sym *sym)
 {
+  int rv;
   size_t nbits = sym->nbits;
+
   for(;;) {
     if(rembits > nbits) {
-      **dest_ptr |= sym->code << (rembits - nbits);
+      if(*avail_ptr) {
+        nghttp2_bufs_fast_orb_hold(bufs, sym->code << (rembits - nbits));
+      } else {
+        rv = nghttp2_bufs_orb_hold(bufs, sym->code << (rembits - nbits));
+        if(rv != 0) {
+          return rv;
+        }
+
+        *avail_ptr = nghttp2_bufs_cur_avail(bufs);
+      }
+
       rembits -= nbits;
+
       break;
     }
-    **dest_ptr |= sym->code >> (nbits - rembits);
-    ++*dest_ptr;
+
+    if(*avail_ptr) {
+      nghttp2_bufs_fast_orb(bufs, sym->code >> (nbits - rembits));
+      --*avail_ptr;
+    } else {
+      rv = nghttp2_bufs_orb(bufs, sym->code >> (nbits - rembits));
+      if(rv != 0) {
+        return rv;
+      }
+
+      *avail_ptr = nghttp2_bufs_cur_avail(bufs);
+    }
+
     nbits -= rembits;
     rembits = 8;
+
     if(nbits == 0) {
       break;
     }
-    **dest_ptr = 0;
+
+    if(*avail_ptr) {
+      nghttp2_bufs_fast_addb_hold(bufs, 0);
+    } else {
+      rv = nghttp2_bufs_addb_hold(bufs, 0);
+      if(rv != 0) {
+        return rv;
+      }
+
+      *avail_ptr = nghttp2_bufs_cur_avail(bufs);
+    }
   }
   return rembits;
 }
@@ -74,27 +110,50 @@ size_t nghttp2_hd_huff_encode_count(const uint8_t *src, size_t len)
   return (nbits + 7) / 8;
 }
 
-ssize_t nghttp2_hd_huff_encode(uint8_t *dest, size_t destlen,
+int nghttp2_hd_huff_encode(nghttp2_bufs *bufs,
                            const uint8_t *src, size_t srclen)
 {
+  int rv;
   int rembits = 8;
-  uint8_t *dest_first = dest;
   size_t i;
+  size_t avail;
+
+  avail = nghttp2_bufs_cur_avail(bufs);
 
   for(i = 0; i < srclen; ++i) {
     const nghttp2_huff_sym *sym = &huff_sym_table[src[i]];
     if(rembits == 8) {
-      *dest = 0;
+      if(avail) {
+        nghttp2_bufs_fast_addb_hold(bufs, 0);
+      } else {
+        rv = nghttp2_bufs_addb_hold(bufs, 0);
+        if(rv != 0) {
+          return rv;
+        }
+        avail = nghttp2_bufs_cur_avail(bufs);
+      }
+    }
+    rembits = huff_encode_sym(bufs, &avail, rembits, sym);
+    if(rembits < 0) {
+      return rembits;
     }
-    rembits = huff_encode_sym(&dest, rembits, sym);
   }
   /* 256 is special terminal symbol, pad with its prefix */
   if(rembits < 8) {
     const nghttp2_huff_sym *sym = &huff_sym_table[256];
-    *dest |= sym->code >> (sym->nbits - rembits);
-    ++dest;
+
+    /* Caution we no longer adjust avail here */
+    if(avail) {
+      nghttp2_bufs_fast_orb(bufs, sym->code >> (sym->nbits - rembits));
+    } else {
+      rv = nghttp2_bufs_orb(bufs, sym->code >> (sym->nbits - rembits));
+      if(rv != 0) {
+        return rv;
+      }
     }
-  return dest - dest_first;
+  }
+
+  return 0;
 }
 
 void nghttp2_hd_huff_decode_context_init(nghttp2_hd_huff_decode_context *ctx)
@@ -109,6 +168,10 @@ ssize_t nghttp2_hd_huff_decode(nghttp2_hd_huff_decode_context *ctx,
 {
   size_t i, j;
   int rv;
+  size_t avail;
+
+  avail = nghttp2_bufs_cur_avail(bufs);
+
   /* We use the decoding algorithm described in
      http://graphics.ics.uci.edu/pub/Prefix.pdf */
   for(i = 0; i < srclen; ++i) {
@@ -119,10 +182,16 @@ ssize_t nghttp2_hd_huff_decode(nghttp2_hd_huff_decode_context *ctx,
         return NGHTTP2_ERR_HEADER_COMP;
       }
       if(t->flags & NGHTTP2_HUFF_SYM) {
+        if(avail) {
+          nghttp2_bufs_fast_addb(bufs, t->sym);
+          --avail;
+        } else {
           rv = nghttp2_bufs_addb(bufs, t->sym);
           if(rv != 0) {
             return rv;
           }
+          avail = nghttp2_bufs_cur_avail(bufs);
+        }
       }
       ctx->state = t->state;
       ctx->accept = (t->flags & NGHTTP2_HUFF_ACCEPTED) != 0;

lib/nghttp2_session.h

@@ -55,15 +55,13 @@ typedef enum {
 typedef struct {
   nghttp2_outbound_item *item;
 
-  nghttp2_buf framebuf;
+  nghttp2_bufs framebufs;
   nghttp2_outbound_state state;
 } nghttp2_active_outbound_item;
 
 /* Buffer length for inbound raw byte stream. */
 #define NGHTTP2_INBOUND_BUFFER_LENGTH 16384
 
-#define NGHTTP2_INITIAL_OUTBOUND_FRAMEBUF_LENGTH 16384
-
 #define NGHTTP2_INITIAL_NV_BUFFER_LENGTH 4096
 
 /* Internal state when receiving incoming frame */
@@ -526,7 +524,7 @@ nghttp2_stream* nghttp2_session_get_stream(nghttp2_session *session,
  *     The read_callback failed (session error).
  */
 ssize_t nghttp2_session_pack_data(nghttp2_session *session,
-                                  nghttp2_buf *buf,
+                                  nghttp2_bufs *bufs,
                                   size_t datamax,
                                   nghttp2_private_data *frame);
 

python/cnghttp2.pxd

@@ -290,8 +290,8 @@ cdef extern from 'nghttp2_hd.h':
     int nghttp2_hd_inflate_change_table_size(nghttp2_hd_inflater *inflater,
                                              size_t hd_table_bufsize_max)
 
-    ssize_t nghttp2_hd_deflate_hd(nghttp2_hd_deflater *deflater,
-                                  nghttp2_buf *buf,
+    int nghttp2_hd_deflate_hd(nghttp2_hd_deflater *deflater,
+                              nghttp2_bufs *bufs,
                               nghttp2_nv *nva, size_t nvlen)
 
     ssize_t nghttp2_hd_inflate_hd(nghttp2_hd_inflater *inflater,
@@ -305,11 +305,12 @@ cdef extern from 'nghttp2_hd.h':
 
 cdef extern from 'nghttp2_buf.h':
 
-    ctypedef struct nghttp2_buf:
-        uint8_t *pos
-        uint8_t *last
+    ctypedef struct nghttp2_bufs:
+        pass
 
+    void nghttp2_bufs_init(nghttp2_bufs *bufs, size_t chunk_size,
+                           size_t max_chunk)
 
-    void nghttp2_buf_init(nghttp2_buf *buf)
+    void nghttp2_bufs_free(nghttp2_bufs *bufs)
 
-    void nghttp2_buf_free(nghttp2_buf *buf)
+    ssize_t nghttp2_bufs_remove(nghttp2_bufs *bufs, uint8_t **out)

python/nghttp2.pyx

@@ -114,27 +114,36 @@ cdef class HDDeflater:
             nvap[0].valuelen = len(v)
             nvap += 1
 
-        cdef cnghttp2.nghttp2_buf buf
+        cdef cnghttp2.nghttp2_bufs bufs
         cdef size_t outcap = 0
         cdef ssize_t rv
+        cdef uint8_t *out
 
-        cnghttp2.nghttp2_buf_init(&buf)
+        cnghttp2.nghttp2_bufs_init(&bufs, 4096, 16)
 
-        rv = cnghttp2.nghttp2_hd_deflate_hd(&self._deflater, &buf,
+        rv = cnghttp2.nghttp2_hd_deflate_hd(&self._deflater, &bufs,
                                             nva, len(headers))
         free(nva)
 
         if rv < 0:
-            cnghttp2.nghttp2_buf_free(&buf);
+            cnghttp2.nghttp2_bufs_free(&bufs);
+
+            raise Exception(_strerror(rv))
+
+        rv = cnghttp2.nghttp2_bufs_remove(&bufs, &out)
+
+        if rv < 0:
+            cnghttp2.nghttp2_bufs_free(&bufs);
 
             raise Exception(_strerror(rv))
 
         cdef bytes res
 
         try:
-            res = buf.pos[:rv]
+            res = out[:rv]
         finally:
-            cnghttp2.nghttp2_buf_free(&buf)
+            cnghttp2.nghttp2_free(out)
+            cnghttp2.nghttp2_bufs_free(&bufs)
 
         return res
 

src/app_helper.cc

@@ -324,7 +324,7 @@ void print_frame(print_type ptype, const nghttp2_frame *frame)
   }
   switch(frame->hd.type) {
   case NGHTTP2_DATA:
-    if(frame->hd.flags & (NGHTTP2_FLAG_PAD_HIGH | NGHTTP2_FLAG_PAD_LOW)) {
+    if(frame->data.padlen > 0) {
       print_frame_attr_indent();
       fprintf(outfile, "(padlen=%zu)\n", frame->data.padlen);
     }

src/deflatehd.c

@@ -72,26 +72,36 @@ static void to_hex(char *dest, const uint8_t *src, size_t len)
 }
 
 static void output_to_json(nghttp2_hd_deflater *deflater,
-                           nghttp2_buf *buf, size_t inputlen,
+                           nghttp2_bufs *bufs, size_t inputlen,
                            nghttp2_nv *nva, size_t nvlen,
                            int seq)
 {
   json_t *obj;
-  char *hex = NULL;
+  char *hex = NULL, *hexp;
   size_t len;
+  nghttp2_buf_chain *ci;
+  nghttp2_buf *buf;
 
-  len = nghttp2_buf_len(buf);
+  len = nghttp2_bufs_len(bufs);
 
   if(len > 0) {
     hex = malloc(len * 2);
   }
+
   obj = json_object();
   json_object_set_new(obj, "seq", json_integer(seq));
   json_object_set_new(obj, "input_length", json_integer(inputlen));
   json_object_set_new(obj, "output_length", json_integer(len));
   json_object_set_new(obj, "percentage_of_original_size",
                       json_real((double)len / inputlen * 100));
-  to_hex(hex, buf->pos, len);
+
+  hexp = hex;
+  for(ci = bufs->head; ci; ci = ci->next) {
+    buf = &ci->buf;
+    to_hex(hexp, buf->pos, nghttp2_buf_len(buf));
+    hexp += nghttp2_buf_len(buf);
+  }
+
   if(len == 0) {
     json_object_set_new(obj, "wire", json_string(""));
   } else {
@@ -117,11 +127,11 @@ static void deflate_hd(nghttp2_hd_deflater *deflater,
                        nghttp2_nv *nva, size_t nvlen, size_t inputlen, int seq)
 {
   ssize_t rv;
-  nghttp2_buf buf;
+  nghttp2_bufs bufs;
 
-  nghttp2_buf_init(&buf);
+  nghttp2_bufs_init2(&bufs, 4096, 16, 0);
 
-  rv = nghttp2_hd_deflate_hd(deflater, &buf, nva, nvlen);
+  rv = nghttp2_hd_deflate_hd(deflater, &bufs, nva, nvlen);
   if(rv < 0) {
     fprintf(stderr, "deflate failed with error code %zd at %d\n", rv, seq);
     exit(EXIT_FAILURE);
@@ -130,8 +140,8 @@ static void deflate_hd(nghttp2_hd_deflater *deflater,
   input_sum += inputlen;
   output_sum += rv;
 
-  output_to_json(deflater, &buf, inputlen, nva, nvlen, seq);
-  nghttp2_buf_free(&buf);
+  output_to_json(deflater, &bufs, inputlen, nva, nvlen, seq);
+  nghttp2_bufs_free(&bufs);
 }
 
 static int deflate_hd_json(json_t *obj, nghttp2_hd_deflater *deflater, int seq)

tests/nghttp2_test_helper.c

@@ -28,8 +28,15 @@
 
 #include <CUnit/CUnit.h>
 
-int unpack_framebuf(nghttp2_frame *frame, nghttp2_buf *buf)
+#include "nghttp2_helper.h"
+
+int unpack_framebuf(nghttp2_frame *frame, nghttp2_bufs *bufs)
 {
+  nghttp2_buf *buf;
+
+  /* Assuming we have required data in first buffer. We don't decode
+     header block so, we don't mind its space */
+  buf = &bufs->head->buf;
   return unpack_frame(frame, buf->pos, nghttp2_buf_len(buf));
 }
 
@@ -143,21 +150,44 @@ void add_out(nva_out *out, nghttp2_nv *nv)
 }
 
 ssize_t inflate_hd(nghttp2_hd_inflater *inflater, nva_out *out,
-                   uint8_t *buf, size_t buflen)
+                   nghttp2_bufs *bufs, size_t offset)
 {
   ssize_t rv;
   nghttp2_nv nv;
   int inflate_flags;
-  size_t initial = buflen;
+  nghttp2_buf_chain *ci;
+  nghttp2_buf *buf;
+  nghttp2_buf bp;
+  int final;
+  size_t processed;
+
+  processed = 0;
+
+  for(ci = bufs->head; ci; ci = ci->next) {
+    buf = &ci->buf;
+    final = nghttp2_buf_len(buf) == 0 || ci->next == NULL;
+    bp = *buf;
+
+    if(offset) {
+      int n;
+
+      n = nghttp2_min((ssize_t)offset, nghttp2_buf_len(&bp));
+      bp.pos += n;
+      offset -= n;
+    }
 
     for(;;) {
       inflate_flags = 0;
-    rv = nghttp2_hd_inflate_hd(inflater, &nv, &inflate_flags, buf, buflen, 1);
+      rv = nghttp2_hd_inflate_hd(inflater, &nv, &inflate_flags,
+                                 bp.pos, nghttp2_buf_len(&bp), final);
+
       if(rv < 0) {
         return rv;
       }
-    buf += rv;
-    buflen -= rv;
+
+      bp.pos += rv;
+      processed += rv;
+
       if(inflate_flags & NGHTTP2_HD_INFLATE_EMIT) {
         add_out(out, &nv);
       }
@@ -165,6 +195,21 @@ ssize_t inflate_hd(nghttp2_hd_inflater *inflater, nva_out *out,
         break;
       }
     }
+  }
+
   nghttp2_hd_inflate_end_headers(inflater);
-  return initial - buflen;
+
+  return processed;
+}
+
+void frame_pack_bufs_init(nghttp2_bufs *bufs)
+{
+  /* 2 for PAD_HIGH and PAD_LOW */
+  nghttp2_bufs_init2(bufs, 4096, 16, NGHTTP2_FRAME_HDLEN + 2);
+}
+
+void bufs_large_init(nghttp2_bufs *bufs, size_t chunk_size)
+{
+  /* 2 for PAD_HIGH and PAD_LOW */
+  nghttp2_bufs_init2(bufs, chunk_size, 16, NGHTTP2_FRAME_HDLEN + 2);
 }

tests/nghttp2_test_helper.h

@@ -54,7 +54,7 @@
     free(a);                                                \
   } while(0);
 
-int unpack_framebuf(nghttp2_frame *frame, nghttp2_buf *buf);
+int unpack_framebuf(nghttp2_frame *frame, nghttp2_bufs *bufs);
 
 int unpack_frame(nghttp2_frame *frame, const uint8_t *in, size_t len);
 
@@ -75,6 +75,10 @@ void nva_out_reset(nva_out *out);
 void add_out(nva_out *out, nghttp2_nv *nv);
 
 ssize_t inflate_hd(nghttp2_hd_inflater *inflater, nva_out *out,
-                   uint8_t *buf, size_t buflen);
+                   nghttp2_bufs *bufs, size_t offset);
+
+void frame_pack_bufs_init(nghttp2_bufs *bufs);
+
+void bufs_large_init(nghttp2_bufs *bufs, size_t chunk_size);
 
 #endif /* NGHTTP2_TEST_HELPER_H */