Introduce nghttp2_buf to ease buffer management

lib/Makefile.am

@@ -33,6 +33,7 @@ lib_LTLIBRARIES = libnghttp2.la
 
 OBJECTS = nghttp2_pq.c nghttp2_map.c nghttp2_queue.c \
 	nghttp2_buffer.c nghttp2_frame.c \
+	nghttp2_buf.c \
 	nghttp2_stream.c nghttp2_outbound_item.c \
 	nghttp2_session.c nghttp2_submit.c \
 	nghttp2_helper.c \
@@ -42,6 +43,7 @@ OBJECTS = nghttp2_pq.c nghttp2_map.c nghttp2_queue.c \
 
 HFILES = nghttp2_pq.h nghttp2_int.h nghttp2_map.h nghttp2_queue.h \
 	nghttp2_buffer.h nghttp2_frame.h \
+	nghttp2_buf.h \
 	nghttp2_session.h nghttp2_helper.h nghttp2_stream.h nghttp2_int.h \
 	nghttp2_npn.h nghttp2_gzip.h \
 	nghttp2_submit.h nghttp2_outbound_item.h \

lib/nghttp2_buf.c

+/*
+ * nghttp2 - HTTP/2.0 C Library
+ *
+ * Copyright (c) 2014 Tatsuhiro Tsujikawa
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "nghttp2_buf.h"
+
+#include "nghttp2_helper.h"
+
+void nghttp2_buf_init(nghttp2_buf *buf)
+{
+  buf->begin = NULL;
+  buf->end = NULL;
+  buf->pos = NULL;
+  buf->last = NULL;
+  buf->mark = NULL;
+}
+
+int nghttp2_buf_init2(nghttp2_buf *buf, size_t initial)
+{
+  nghttp2_buf_init(buf);
+  return nghttp2_buf_reserve(buf, initial);
+}
+
+void nghttp2_buf_free(nghttp2_buf *buf)
+{
+  free(buf->begin);
+}
+
+int nghttp2_buf_reserve(nghttp2_buf *buf, size_t new_cap)
+{
+  uint8_t *ptr;
+  size_t cap;
+
+  cap = nghttp2_buf_cap(buf);
+
+  if(cap >= new_cap) {
+    return 0;
+  }
+
+  new_cap = nghttp2_max(new_cap, cap * 2);
+
+  ptr = realloc(buf->begin, new_cap);
+  if(ptr == NULL) {
+    return NGHTTP2_ERR_NOMEM;
+  }
+
+  buf->pos = ptr + (buf->pos - buf->begin);
+  buf->last = ptr + (buf->last - buf->begin);
+  buf->mark = ptr + (buf->mark - buf->begin);
+  buf->begin = ptr;
+  buf->end = ptr + new_cap;
+
+  return 0;
+}
+
+int nghttp2_buf_pos_reserve(nghttp2_buf *buf, size_t new_rel_cap)
+{
+  return nghttp2_buf_reserve(buf, nghttp2_buf_pos_offset(buf) + new_rel_cap);
+}
+
+int nghttp2_buf_last_reserve(nghttp2_buf *buf, size_t new_rel_cap)
+{
+  return nghttp2_buf_reserve(buf, nghttp2_buf_last_offset(buf) + new_rel_cap);
+}
+
+void nghttp2_buf_reset(nghttp2_buf *buf)
+{
+  buf->pos = buf->last = buf->mark = buf->begin;
+}
+
+void nghttp2_buf_wrap_init(nghttp2_buf *buf, uint8_t *begin, size_t len)
+{
+  buf->begin = buf->pos = buf->last = buf->mark = begin;
+  buf->end = begin + len;
+}

lib/nghttp2_buf.h

+/*
+ * nghttp2 - HTTP/2.0 C Library
+ *
+ * Copyright (c) 2014 Tatsuhiro Tsujikawa
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+#ifndef NGHTTP2_BUF_H
+#define NGHTTP2_BUF_H
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif /* HAVE_CONFIG_H */
+
+#include <nghttp2/nghttp2.h>
+
+#include "nghttp2_int.h"
+
+typedef struct {
+  /* This points to the beginning of the buffer. The effective range
+     of buffer is [begin, end). */
+  uint8_t *begin;
+  /* This points to the memory one byte beyond the end of the
+     buffer. */
+  uint8_t *end;
+  /* The position indicator for effective start of the buffer. pos <=
+     last must be hold. */
+  uint8_t *pos;
+  /* The position indicator for effective one beyond of the end of the
+     buffer. last <= end must be hold. */
+  uint8_t *last;
+  /* Mark arbitrary position in buffer [begin, end) */
+  uint8_t *mark;
+} nghttp2_buf;
+
+#define nghttp2_buf_len(BUF) ((BUF)->last - (BUF)->pos)
+#define nghttp2_buf_avail(BUF) ((BUF)->end - (BUF)->last)
+#define nghttp2_buf_cap(BUF) ((BUF)->end - (BUF)->begin)
+
+#define nghttp2_buf_pos_offset(BUF) ((BUF)->pos - (BUF)->begin)
+#define nghttp2_buf_last_offset(BUF) ((BUF)->last - (BUF)->begin)
+
+#define nghttp2_buf_shift_right(BUF, AMT) \
+  do {                                    \
+    (BUF)->pos += AMT;                    \
+    (BUF)->last += AMT;                   \
+  } while(0)
+
+#define nghttp2_buf_shift_left(BUF, AMT)            \
+  do {                                              \
+    (BUF)->pos -= AMT;                              \
+    (BUF)->last -= AMT;                             \
+  } while(0)
+
+/*
+ * Initializes the |buf|. No memory is allocated in this function. Use
+ * nghttp2_buf_reserve() or nghttp2_buf_reserve2() to allocate memory.
+ */
+void nghttp2_buf_init(nghttp2_buf *buf);
+
+
+/*
+ * Initializes the |buf| and allocates at least |initial| bytes of
+ * memory.
+ *
+ * This function returns 0 if it succeeds, or one of the following
+ * negative error codes:
+ *
+ * NGHTTP2_ERR_NOMEM
+ *     Out of memory
+ */
+int nghttp2_buf_init2(nghttp2_buf *buf, size_t initial);
+
+/*
+ * Frees buffer in |buf|.
+ */
+void nghttp2_buf_free(nghttp2_buf *buf);
+
+/*
+ * Extends buffer so that nghttp2_buf_cap() returns at least
+ * |new_cap|. If extensions took place, buffer pointers in |buf| will
+ * change.
+ *
+ * This function returns 0 if it succeeds, or one of the followings
+ * negative error codes:
+ *
+ * NGHTTP2_ERR_NOMEM
+ *     Out of memory
+ */
+int nghttp2_buf_reserve(nghttp2_buf *buf, size_t new_cap);
+
+/*
+ * This function behaves like nghttp2_buf_reserve(), but new capacity
+ * is calculated as nghttp2_buf_pos_offset(buf) + new_rel_cap. In
+ * other words, this function reserves memory at least |new_rel_cap|
+ * bytes from buf->pos.
+ */
+int nghttp2_buf_pos_reserve(nghttp2_buf *buf, size_t new_rel_cap);
+
+/*
+ * This function behaves like nghttp2_buf_reserve(), but new capacity
+ * is calculated as nghttp2_buf_last_offset(buf) + new_rel_cap. In
+ * other words, this function reserves memory at least |new_rel_cap|
+ * bytes from buf->last.
+ */
+int nghttp2_buf_last_reserve(nghttp2_buf *buf, size_t new_rel_cap);
+
+/*
+ * Resets pos, last, mark member of |buf| to buf->begin.
+ */
+void nghttp2_buf_reset(nghttp2_buf *buf);
+
+/*
+ * Initializes |buf| using supplied buffer |begin| of length
+ * |len|. Semantically, the application should not call *_reserve() or
+ * nghttp2_free() functions for |buf|.
+ */
+void nghttp2_buf_wrap_init(nghttp2_buf *buf, uint8_t *begin, size_t len);
+
+#endif /* NGHTTP2_BUF_H */

lib/nghttp2_hd.h

@@ -33,6 +33,7 @@
 
 #include "nghttp2_hd_huffman.h"
 #include "nghttp2_buffer.h"
+#include "nghttp2_buf.h"
 
 #define NGHTTP2_HD_DEFAULT_MAX_BUFFER_SIZE (1 << 12)
 #define NGHTTP2_HD_ENTRY_OVERHEAD 32
@@ -295,16 +296,13 @@ int nghttp2_hd_inflate_change_table_size(nghttp2_hd_inflater *inflater,
 
 /*
  * Deflates the |nva|, which has the |nvlen| name/value pairs, into
- * the buffer pointed by the |*buf_ptr| with the length |*buflen_ptr|.
- * The output starts after |nv_offset| bytes from |*buf_ptr|.
+ * the buffer pointed by the |buf|. The caller must ensure that
+ * nghttp2_buf_len(buf) == 0 holds. Write starts at buf->last.
  *
- * This function expands |*buf_ptr| as necessary to store the
- * result. When expansion occurred, memory previously pointed by
- * |*buf_ptr| may change.  |*buf_ptr| and |*buflen_ptr| are updated
- * accordingly.
+ * This function expands |buf| as necessary to store the result.
  *
- * This function copies necessary data into |*buf_ptr|. After this
- * function returns, it is safe to delete the |nva|.
+ * This function copies necessary data into |buf|. After this function
+ * returns, it is safe to delete the |nva|.
  *
  * TODO: The rest of the code call nghttp2_hd_end_headers() after this
  * call, but it is just a regacy of the first implementation. Now it
@@ -319,8 +317,7 @@ int nghttp2_hd_inflate_change_table_size(nghttp2_hd_inflater *inflater,
  *     Deflation process has failed.
  */
 ssize_t nghttp2_hd_deflate_hd(nghttp2_hd_deflater *deflater,
-                              uint8_t **buf_ptr, size_t *buflen_ptr,
-                              size_t nv_offset,
+                              nghttp2_buf *buf,
                               nghttp2_nv *nva, size_t nvlen);
 
 typedef enum {
@@ -373,19 +370,16 @@ ssize_t nghttp2_hd_inflate_hd(nghttp2_hd_inflater *inflater,
 int nghttp2_hd_inflate_end_headers(nghttp2_hd_inflater *inflater);
 
 /* For unittesting purpose */
-int nghttp2_hd_emit_indname_block(uint8_t **buf_ptr, size_t *buflen_ptr,
-                                  size_t *offset_ptr, size_t index,
+int nghttp2_hd_emit_indname_block(nghttp2_buf *buf, size_t index,
                                   const uint8_t *value, size_t valuelen,
                                   int inc_indexing);
 
 /* For unittesting purpose */
-int nghttp2_hd_emit_newname_block(uint8_t **buf_ptr, size_t *buflen_ptr,
-                                  size_t *offset_ptr, nghttp2_nv *nv,
+int nghttp2_hd_emit_newname_block(nghttp2_buf *buf, nghttp2_nv *nv,
                                   int inc_indexing);
 
 /* For unittesting purpose */
-int nghttp2_hd_emit_table_size(uint8_t **buf_ptr, size_t *buflen_ptr,
-                               size_t *offset_ptr, size_t table_size);
+int nghttp2_hd_emit_table_size(nghttp2_buf *buf, size_t table_size);
 
 /* For unittesting purpose */
 nghttp2_hd_entry* nghttp2_hd_table_get(nghttp2_hd_context *context,

lib/nghttp2_helper.c

@@ -382,3 +382,10 @@ int nghttp2_check_header_value(const uint8_t *value, size_t len)
   }
   return 1;
 }
+
+uint8_t* nghttp2_cpymem(uint8_t *dest, uint8_t *src, size_t len)
+{
+  memcpy(dest, src, len);
+
+  return dest + len;
+}

lib/nghttp2_helper.h

@@ -126,4 +126,11 @@ int nghttp2_should_send_window_update(int32_t local_window_size,
  */
 void nghttp2_free(void *ptr);
 
+/*
+ * Copies the buffer |src| of length |len| to the destination pointed
+ * by the |dest|, assuming that the |dest| is at lest |len| bytes long
+ * . Returns dest + len.
+ */
+uint8_t* nghttp2_cpymem(uint8_t *dest, uint8_t *src, size_t len);
+
 #endif /* NGHTTP2_HELPER_H */

lib/nghttp2_session.h

@@ -38,6 +38,7 @@
 #include "nghttp2_buffer.h"
 #include "nghttp2_outbound_item.h"
 #include "nghttp2_int.h"
+#include "nghttp2_buf.h"
 
 /*
  * Option flags.
@@ -54,27 +55,15 @@ typedef enum {
 
 typedef struct {
   nghttp2_outbound_item *item;
-  /* Buffer for outbound frames. Used to pack one frame. The memory
-     pointed by framebuf is initially allocated by
-     nghttp2_session_{client,server}_new() and deallocated by
-     nghttp2_session_del() */
-  uint8_t *framebuf;
-  /* The capacity of framebuf in bytes */
-  size_t framebufmax;
-  /* The length of the frame stored in framebuf */
-  size_t framebuflen;
-  /* The number of bytes has been sent */
-  size_t framebufoff;
-  /* Marks the last position to send. This is used to implement
-     CONTINUATION */
-  size_t framebufmark;
+
+  nghttp2_buf framebuf;
   nghttp2_outbound_state state;
 } nghttp2_active_outbound_item;
 
 /* Buffer length for inbound raw byte stream. */
 #define NGHTTP2_INBOUND_BUFFER_LENGTH 16384
 
-#define NGHTTP2_INITIAL_OUTBOUND_FRAMEBUF_LENGTH 4096
+#define NGHTTP2_INITIAL_OUTBOUND_FRAMEBUF_LENGTH 16384
 
 #define NGHTTP2_INITIAL_NV_BUFFER_LENGTH 4096
 
@@ -538,8 +527,7 @@ nghttp2_stream* nghttp2_session_get_stream(nghttp2_session *session,
  *     The read_callback failed (session error).
  */
 ssize_t nghttp2_session_pack_data(nghttp2_session *session,
-                                  uint8_t **buf_ptr, size_t *buflen_ptr,
-                                  size_t *bufoff_ptr,
+                                  nghttp2_buf *buf,
                                   size_t datamax,
                                   nghttp2_private_data *frame);
 

python/cnghttp2.pxd

@@ -291,8 +291,7 @@ cdef extern from 'nghttp2_hd.h':
                                              size_t hd_table_bufsize_max)
 
     ssize_t nghttp2_hd_deflate_hd(nghttp2_hd_deflater *deflater,
-                                  uint8_t **buf_ptr, size_t *buflen_ptr,
-                                  size_t nv_offset,
+                                  nghttp2_buf *buf,
                                   nghttp2_nv *nva, size_t nvlen)
 
     ssize_t nghttp2_hd_inflate_hd(nghttp2_hd_inflater *inflater,
@@ -303,3 +302,14 @@ cdef extern from 'nghttp2_hd.h':
 
     nghttp2_hd_entry* nghttp2_hd_table_get(nghttp2_hd_context *context,
                                            size_t index)
+
+cdef extern from 'nghttp2_buf.h':
+
+    ctypedef struct nghttp2_buf:
+        uint8_t *pos
+        uint8_t *last
+
+
+    void nghttp2_buf_init(nghttp2_buf *buf)
+
+    void nghttp2_buf_free(nghttp2_buf *buf)

python/nghttp2.pyx

@@ -106,25 +106,36 @@ cdef class HDDeflater:
                                         malloc(sizeof(cnghttp2.nghttp2_nv)*\
                                         len(headers))
         cdef cnghttp2.nghttp2_nv *nvap = nva
+
         for k, v in headers:
             nvap[0].name = k
             nvap[0].namelen = len(k)
             nvap[0].value = v
             nvap[0].valuelen = len(v)
             nvap += 1
-        cdef uint8_t *out = NULL
+
+        cdef cnghttp2.nghttp2_buf buf
         cdef size_t outcap = 0
         cdef ssize_t rv
-        rv = cnghttp2.nghttp2_hd_deflate_hd(&self._deflater, &out, &outcap,
-                                            0, nva, len(headers))
+
+        cnghttp2.nghttp2_buf_init(&buf)
+
+        rv = cnghttp2.nghttp2_hd_deflate_hd(&self._deflater, &buf,
+                                            nva, len(headers))
         free(nva)
+
         if rv < 0:
+            cnghttp2.nghttp2_buf_free(&buf);
+
             raise Exception(_strerror(rv))
+
         cdef bytes res
+
         try:
-            res = out[:rv]
+            res = buf.pos[:rv]
         finally:
-            cnghttp2.nghttp2_free(out)
+            cnghttp2.nghttp2_buf_free(&buf)
+
         return res
 
     def set_no_refset(self, no_refset):

src/deflatehd.c

@@ -72,12 +72,15 @@ static void to_hex(char *dest, const uint8_t *src, size_t len)
 }
 
 static void output_to_json(nghttp2_hd_deflater *deflater,
-                           const uint8_t *buf, size_t len, size_t inputlen,
+                           nghttp2_buf *buf, size_t inputlen,
                            nghttp2_nv *nva, size_t nvlen,
                            int seq)
 {
   json_t *obj;
   char *hex = NULL;
+  size_t len;
+
+  len = nghttp2_buf_len(buf);
 
   if(len > 0) {
     hex = malloc(len * 2);
@@ -88,7 +91,7 @@ static void output_to_json(nghttp2_hd_deflater *deflater,
   json_object_set_new(obj, "output_length", json_integer(len));
   json_object_set_new(obj, "percentage_of_original_size",
                       json_real((double)len / inputlen * 100));
-  to_hex(hex, buf, len);
+  to_hex(hex, buf->pos, len);
   if(len == 0) {
     json_object_set_new(obj, "wire", json_string(""));
   } else {
@@ -114,17 +117,21 @@ static void deflate_hd(nghttp2_hd_deflater *deflater,
                        nghttp2_nv *nva, size_t nvlen, size_t inputlen, int seq)
 {
   ssize_t rv;
-  uint8_t *buf = NULL;
-  size_t buflen = 0;
-  rv = nghttp2_hd_deflate_hd(deflater, &buf, &buflen, 0, nva, nvlen);
+  nghttp2_buf buf;
+
+  nghttp2_buf_init(&buf);
+
+  rv = nghttp2_hd_deflate_hd(deflater, &buf, nva, nvlen);
   if(rv < 0) {
     fprintf(stderr, "deflate failed with error code %zd at %d\n", rv, seq);
     exit(EXIT_FAILURE);
   }
+
   input_sum += inputlen;
   output_sum += rv;
-  output_to_json(deflater, buf, rv, inputlen, nva, nvlen, seq);
-  free(buf);
+
+  output_to_json(deflater, &buf, inputlen, nva, nvlen, seq);
+  nghttp2_buf_free(&buf);
 }
 
 static int deflate_hd_json(json_t *obj, nghttp2_hd_deflater *deflater, int seq)

tests/main.c

@@ -202,6 +202,12 @@ int main(int argc, char* argv[])
                    test_nghttp2_session_pack_data_with_padding) ||
       !CU_add_test(pSuite, "session_pack_headers_with_padding",
                    test_nghttp2_session_pack_headers_with_padding) ||
+      !CU_add_test(pSuite, "session_pack_headers_with_padding2",
+                   test_nghttp2_session_pack_headers_with_padding2) ||
+      !CU_add_test(pSuite, "session_pack_headers_with_padding3",
+                   test_nghttp2_session_pack_headers_with_padding3) ||
+      !CU_add_test(pSuite, "session_pack_headers_with_padding4",
+                   test_nghttp2_session_pack_headers_with_padding4) ||
       !CU_add_test(pSuite, "pack_settings_payload",
                    test_nghttp2_pack_settings_payload) ||
       !CU_add_test(pSuite, "frame_pack_headers",

tests/nghttp2_session_test.h

@@ -92,6 +92,9 @@ void test_nghttp2_session_set_option(void);
 void test_nghttp2_session_data_backoff_by_high_pri_frame(void);
 void test_nghttp2_session_pack_data_with_padding(void);
 void test_nghttp2_session_pack_headers_with_padding(void);
+void test_nghttp2_session_pack_headers_with_padding2(void);
+void test_nghttp2_session_pack_headers_with_padding3(void);
+void test_nghttp2_session_pack_headers_with_padding4(void);
 void test_nghttp2_pack_settings_payload(void);
 
 #endif /* NGHTTP2_SESSION_TEST_H */

tests/nghttp2_test_helper.c

@@ -28,6 +28,11 @@
 
 #include <CUnit/CUnit.h>
 
+int unpack_framebuf(nghttp2_frame *frame, nghttp2_buf *buf)
+{
+  return unpack_frame(frame, buf->pos, nghttp2_buf_len(buf));
+}
+
 int unpack_frame(nghttp2_frame *frame, const uint8_t *in, size_t len)
 {
   ssize_t rv = 0;

tests/nghttp2_test_helper.h

@@ -54,6 +54,8 @@
     free(a);                                                \
   } while(0);
 
+int unpack_framebuf(nghttp2_frame *frame, nghttp2_buf *buf);
+
 int unpack_frame(nghttp2_frame *frame, const uint8_t *in, size_t len);
 
 int strmemeq(const char *a, const uint8_t *b, size_t bn);