Use libev for nghttpd

Benchmark shows 10% faster with libev compared to libevent.  Also
response time in high load condition is much faster.

configure.ac

@@ -275,6 +275,21 @@ fi
 
 AM_CONDITIONAL([HAVE_CUNIT], [ test "x${have_cunit}" = "xyes" ])
 
+# libev (for src)
+# libev does not have pkg-config file.  Check it in an old way.
+LIBS_OLD=$LIBS
+AC_CHECK_LIB([ev], [ev_time], [have_libev=yes], [have_libev=no])
+if test "x${have_libev}" = "xyes"; then
+  AC_CHECK_HEADER([ev.h], [have_libev=yes], [have_libev=no])
+  if test "x${have_libev}" = "xyes"; then
+    LIBEV_LIBS=-lev
+    LIBEV_CFLAGS=
+    AC_SUBST([LIBEV_LIBS])
+    AC_SUBST([LIBEV_CFLAGS])
+  fi
+fi
+LIBS=$LIBS_OLD
+
 # openssl (for src)
 PKG_CHECK_MODULES([OPENSSL], [openssl >= 1.0.1],
                   [have_openssl=yes], [have_openssl=no])
@@ -375,11 +390,12 @@ if test "x${request_asio_lib}" = "xyes"; then
 fi
 
 # The nghttp, nghttpd and nghttpx under src depend on zlib, OpenSSL
-# and libevent_openssl
+# libev and libevent_openssl
 enable_app=no
 if test "x${request_app}" != "xno" &&
    test "x${have_zlib}" = "xyes" &&
    test "x${have_openssl}" = "xyes" &&
+   test "x${have_libev}" = "xyes" &&
    test "x${have_libevent_openssl}" = "xyes"; then
   enable_app=yes
 fi
@@ -649,6 +665,7 @@ AC_MSG_NOTICE([summary of build options:
     Libs:
       OpenSSL:        ${have_openssl}
       Libxml2:        ${have_libxml2}
+      Libev:          ${have_libev}
       Libevent(SSL):  ${have_libevent_openssl}
       Spdylay:        ${have_spdylay}
       Jansson:        ${have_jansson}

src/HttpServer.h

@@ -39,22 +39,12 @@
 
 #include <openssl/ssl.h>
 
-#include <event2/event.h>
-#include <event2/bufferevent.h>
+#include <ev.h>
 
 #include <nghttp2/nghttp2.h>
 
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "nghttp2_buf.h"
-
-#ifdef __cplusplus
-}
-#endif
-
 #include "http2.h"
+#include "ringbuf.h"
 
 namespace nghttp2 {
 
@@ -65,8 +55,8 @@ struct Config {
   std::string private_key_file;
   std::string cert_file;
   std::string dh_param_file;
-  timeval stream_read_timeout;
-  timeval stream_write_timeout;
+  ev_tstamp stream_read_timeout;
+  ev_tstamp stream_write_timeout;
   nghttp2_option *session_option;
   void *data_ptr;
   size_t padding;
@@ -88,8 +78,8 @@ class Http2Handler;
 struct Stream {
   Headers headers;
   Http2Handler *handler;
-  event *rtimer;
-  event *wtimer;
+  ev_timer rtimer;
+  ev_timer wtimer;
   int64_t body_left;
   int32_t stream_id;
   int file;
@@ -106,7 +96,6 @@ public:
 
   void remove_self();
   int setup_bev();
-  int send();
   int on_read();
   int on_write();
   int on_connect();
@@ -137,14 +126,29 @@ public:
   void remove_settings_timer();
   void terminate_session(uint32_t error_code);
 
+  int fill_rb();
+
+  int read_clear();
+  int write_clear();
+  int tls_handshake();
+  int read_tls();
+  int write_tls();
+
+  struct ev_loop *get_loop() const;
+
 private:
+  ev_io wev_;
+  ev_io rev_;
+  ev_timer settings_timerev_;
   std::map<int32_t, std::unique_ptr<Stream>> id2stream_;
+  RingBuf<16384> rb_;
+  std::function<int(Http2Handler &)> read_, write_;
   int64_t session_id_;
   nghttp2_session *session_;
   Sessions *sessions_;
   SSL *ssl_;
-  bufferevent *bev_;
-  event *settings_timerev_;
+  const uint8_t *data_pending_;
+  size_t data_pendinglen_;
   int fd_;
 };
 

src/Makefile.am

@@ -50,6 +50,15 @@ AM_LDFLAGS = \
 	@JANSSON_LIBS@ \
 	@ZLIB_LIBS@ \
 	@APPLDFLAGS@
+EVLDFLAGS = \
+	@JEMALLOC_LIBS@ \
+	@LIBSPDYLAY_LIBS@ \
+	@XML_LIBS@ \
+	@LIBEV_LIBS@ \
+	@OPENSSL_LIBS@ \
+	@JANSSON_LIBS@ \
+	@ZLIB_LIBS@ \
+	@APPLDFLAGS@
 
 LDADD = \
 	$(top_builddir)/lib/libnghttp2.la \
@@ -59,9 +68,9 @@ if ENABLE_APP
 
 bin_PROGRAMS += nghttp nghttpd nghttpx
 
-HELPER_OBJECTS = util.cc libevent_util.cc \
+HELPER_OBJECTS = util.cc \
 	http2.cc timegm.c app_helper.cc nghttp2_gzip.c
-HELPER_HFILES = util.h libevent_util.h \
+HELPER_HFILES = util.h \
 	http2.h timegm.h app_helper.h nghttp2_config.h \
 	nghttp2_gzip.h
 
@@ -74,11 +83,15 @@ endif # HAVE_LIBXML2
 
 nghttp_SOURCES = ${HELPER_OBJECTS} ${HELPER_HFILES} nghttp.cc \
 	${HTML_PARSER_OBJECTS} ${HTML_PARSER_HFILES} \
+	libevent_util.cc libevent_util.h \
 	ssl.cc ssl.h
 
+nghttpd_LDFLAGS = ${EVLDFLAGS}
+
 nghttpd_SOURCES = ${HELPER_OBJECTS} ${HELPER_HFILES} nghttpd.cc \
 	ssl.cc ssl.h \
-	HttpServer.cc HttpServer.h
+	HttpServer.cc HttpServer.h \
+	ringbuf.h
 
 bin_PROGRAMS += h2load
 
@@ -141,7 +154,8 @@ nghttpx_unittest_SOURCES = shrpx-unittest.cc \
 	http2_test.cc http2_test.h \
 	util_test.cc util_test.h \
 	nghttp2_gzip_test.c nghttp2_gzip_test.h \
-	nghttp2_gzip.c nghttp2_gzip.h
+	nghttp2_gzip.c nghttp2_gzip.h \
+	ringbuf_test.cc ringbuf_test.h
 nghttpx_unittest_CPPFLAGS = ${AM_CPPFLAGS}\
 	 -DNGHTTP2_TESTS_DIR=\"$(top_srcdir)/tests\"
 nghttpx_unittest_LDFLAGS = ${AM_LDFLAGS} \

src/ringbuf.h

+/*
+ * nghttp2 - HTTP/2 C Library
+ *
+ * Copyright (c) 2014 Tatsuhiro Tsujikawa
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+#ifndef RINGBUF_H
+#define RINGBUF_H
+
+#include <sys/uio.h>
+
+#include <cstring>
+#include <algorithm>
+
+namespace nghttp2 {
+
+template <size_t N> struct RingBuf {
+  RingBuf() : pos(0), len(0) {}
+  // Returns the number of bytes to read.
+  size_t rleft() const { return len; }
+  // Returns the number of bytes this buffer can store.
+  size_t wleft() const { return N - len; }
+  // Writes up to min(wleft(), |count|) bytes from buffer pointed by
+  // |buf|.  Returns number of bytes written.
+  size_t write(const void *buf, size_t count) {
+    count = std::min(count, wleft());
+    auto last = (pos + len) % N;
+    if (count > N - last) {
+      auto c = N - last;
+      memcpy(begin + last, buf, c);
+      memcpy(begin, reinterpret_cast<const uint8_t *>(buf) + c, count - c);
+    } else {
+      memcpy(begin + last, buf, count);
+    }
+    len += count;
+    return count;
+  }
+  // Drains min(rleft(), |count|) bytes from start of the buffer.
+  size_t drain(size_t count) {
+    count = std::min(count, rleft());
+    pos = (pos + count) % N;
+    len -= count;
+    return count;
+  }
+  // Returns pointer to the next contiguous readable buffer and its
+  // length.
+  std::pair<const void *, size_t> get() const {
+    if (pos + len > N) {
+      return {begin + pos, N - pos};
+    }
+    return {begin + pos, len};
+  }
+  void reset() { pos = len = 0; }
+  // Fills |iov| for reading.  |iov| must contain at least 2 elements.
+  // Returns the number of filled elements.
+  int riovec(struct iovec *iov) {
+    if (len == 0) {
+      return 0;
+    }
+    if (pos + len > N) {
+      auto c = N - pos;
+      iov[0].iov_base = begin + pos;
+      iov[0].iov_len = c;
+      iov[1].iov_base = begin;
+      iov[1].iov_len = len - c;
+      return 2;
+    }
+    iov[0].iov_base = begin + pos;
+    iov[0].iov_len = len;
+    return 1;
+  }
+  // Fills |iov| for writing.  |iov| must contain at least 2 elements.
+  // Returns the number of filled elements.
+  int wiovec(struct iovec *iov) {
+    if (len == N) {
+      return 0;
+    }
+    if (pos == 0) {
+      iov[0].iov_base = begin + pos + len;
+      iov[0].iov_len = N - pos - len;
+      return 1;
+    }
+    if (pos + len < N) {
+      auto c = N - pos - len;
+      iov[0].iov_base = begin + pos + len;
+      iov[0].iov_len = c;
+      iov[1].iov_base = begin;
+      iov[1].iov_len = N - len - c;
+      return 2;
+    }
+    auto last = (pos + len) % N;
+    iov[0].iov_base = begin + last;
+    iov[0].iov_len = N - len;
+    return 1;
+  }
+  size_t pos;
+  size_t len;
+  uint8_t begin[N];
+};
+
+} // namespace nghttp2
+
+#endif // RINGBUF_H

src/ringbuf_test.cc

+/*
+ * nghttp2 - HTTP/2 C Library
+ *
+ * Copyright (c) 2014 Tatsuhiro Tsujikawa
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "ringbuf_test.h"
+
+#include <cstring>
+#include <iostream>
+
+#include <CUnit/CUnit.h>
+
+#include <nghttp2/nghttp2.h>
+
+#include "ringbuf.h"
+
+namespace nghttp2 {
+
+void test_ringbuf_write(void) {
+  RingBuf<16> b;
+  CU_ASSERT(0 == b.rleft());
+  CU_ASSERT(16 == b.wleft());
+
+  b.write("012", 3);
+
+  CU_ASSERT(3 == b.rleft());
+  CU_ASSERT(13 == b.wleft());
+  CU_ASSERT(0 == b.pos);
+  CU_ASSERT(3 == b.len);
+
+  b.drain(3);
+
+  CU_ASSERT(0 == b.rleft());
+  CU_ASSERT(16 == b.wleft());
+  CU_ASSERT(3 == b.pos);
+  CU_ASSERT(0 == b.len);
+
+  b.write("0123456789ABCDEF", 16);
+
+  CU_ASSERT(16 == b.rleft());
+  CU_ASSERT(0 == b.wleft());
+  CU_ASSERT(3 == b.pos);
+  CU_ASSERT(16 == b.len);
+  CU_ASSERT(0 == memcmp(b.begin, "DEF0123456789ABC", 16));
+
+  const void *p;
+  size_t len;
+  std::tie(p, len) = b.get();
+  CU_ASSERT(13 == len);
+  CU_ASSERT(0 == memcmp(p, "0123456789ABC", len));
+
+  b.drain(14);
+
+  CU_ASSERT(2 == b.rleft());
+  CU_ASSERT(14 == b.wleft());
+  CU_ASSERT(1 == b.pos);
+  CU_ASSERT(2 == b.len);
+
+  std::tie(p, len) = b.get();
+  CU_ASSERT(2 == len);
+  CU_ASSERT(0 == memcmp(p, "EF", len));
+}
+
+void test_ringbuf_iovec(void) {
+  RingBuf<16> b;
+  struct iovec iov[2];
+
+  auto rv = b.riovec(iov);
+
+  CU_ASSERT(0 == rv);
+
+  rv = b.wiovec(iov);
+
+  CU_ASSERT(1 == rv);
+  CU_ASSERT(b.begin == iov[0].iov_base);
+  CU_ASSERT(16 == iov[0].iov_len);
+
+  // set pos to somewhere middle of the buffer, this will require 2
+  // iovec for writing.
+  b.pos = 6;
+
+  rv = b.riovec(iov);
+
+  CU_ASSERT(0 == rv);
+
+  rv = b.wiovec(iov);
+
+  CU_ASSERT(2 == rv);
+  CU_ASSERT(b.begin + b.pos == iov[0].iov_base);
+  CU_ASSERT(10 == iov[0].iov_len);
+  CU_ASSERT(b.begin == iov[1].iov_base);
+  CU_ASSERT(6 == iov[1].iov_len);
+
+  // occupy first region of buffer
+  b.pos = 0;
+  b.len = 10;
+
+  rv = b.riovec(iov);
+
+  CU_ASSERT(1 == rv);
+  CU_ASSERT(b.begin == iov[0].iov_base);
+  CU_ASSERT(10 == iov[0].iov_len);
+
+  rv = b.wiovec(iov);
+
+  CU_ASSERT(1 == rv);
+  CU_ASSERT(b.begin + b.len == iov[0].iov_base);
+  CU_ASSERT(6 == iov[0].iov_len);
+
+  // occupy last region of buffer
+  b.pos = 6;
+  b.len = 10;
+
+  rv = b.riovec(iov);
+
+  CU_ASSERT(1 == rv);
+  CU_ASSERT(b.begin + b.pos == iov[0].iov_base);
+  CU_ASSERT(10 == iov[0].iov_len);
+
+  rv = b.wiovec(iov);
+
+  CU_ASSERT(1 == rv);
+  CU_ASSERT(b.begin == iov[0].iov_base);
+  CU_ASSERT(6 == iov[0].iov_len);
+
+  // occupy middle of buffer
+  b.pos = 3;
+  b.len = 10;
+
+  rv = b.riovec(iov);
+
+  CU_ASSERT(1 == rv);
+  CU_ASSERT(b.begin + b.pos == iov[0].iov_base);
+  CU_ASSERT(10 == iov[0].iov_len);
+
+  rv = b.wiovec(iov);
+
+  CU_ASSERT(2 == rv);
+  CU_ASSERT(b.begin + b.pos + b.len == iov[0].iov_base);
+  CU_ASSERT(3 == iov[0].iov_len);
+  CU_ASSERT(b.begin == iov[1].iov_base);
+  CU_ASSERT(3 == iov[1].iov_len);
+
+  // crossover
+  b.pos = 13;
+  b.len = 10;
+
+  rv = b.riovec(iov);
+
+  CU_ASSERT(2 == rv);
+  CU_ASSERT(b.begin + b.pos == iov[0].iov_base);
+  CU_ASSERT(3 == iov[0].iov_len);
+  CU_ASSERT(b.begin == iov[1].iov_base);
+  CU_ASSERT(7 == iov[1].iov_len);
+
+  rv = b.wiovec(iov);
+
+  CU_ASSERT(1 == rv);
+  CU_ASSERT(b.begin + 7 == iov[0].iov_base);
+  CU_ASSERT(6 == iov[0].iov_len);
+}
+
+} // namespace nghttp2

src/ringbuf_test.h

+/*
+ * nghttp2 - HTTP/2 C Library
+ *
+ * Copyright (c) 2014 Tatsuhiro Tsujikawa
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+#ifndef RINGBUF_TEST_H
+#define RINGBUF_TEST_H
+
+namespace nghttp2 {
+
+void test_ringbuf_write(void);
+void test_ringbuf_iovec(void);
+
+} // namespace nghttp2
+
+#endif // RINGBUF_TEST_H

src/shrpx-unittest.cc

@@ -38,6 +38,7 @@
 #include "http2_test.h"
 #include "util_test.h"
 #include "nghttp2_gzip_test.h"
+#include "ringbuf_test.h"
 
 static int init_suite1(void) { return 0; }
 
@@ -115,7 +116,9 @@ int main(int argc, char *argv[]) {
       !CU_add_test(pSuite, "util_utox", shrpx::test_util_utox) ||
       !CU_add_test(pSuite, "util_http_date", shrpx::test_util_http_date) ||
       !CU_add_test(pSuite, "util_select_h2", shrpx::test_util_select_h2) ||
-      !CU_add_test(pSuite, "gzip_inflate", test_nghttp2_gzip_inflate)) {
+      !CU_add_test(pSuite, "gzip_inflate", test_nghttp2_gzip_inflate) ||
+      !CU_add_test(pSuite, "ringbuf_write", nghttp2::test_ringbuf_write) ||
+      !CU_add_test(pSuite, "ringbuf_iovec", nghttp2::test_ringbuf_iovec)) {
     CU_cleanup_registry();
     return CU_get_error();
   }