nghttpx: Add connection-level flow control window size options

This commit also changes SPDY's flow control size. Previously,
the size for SPDY is the same amount of bytes with HTTP/2.
For example, --frontend-http2-upstream-window-bits=N,
the window size is 2**N - 1. Now SPDY code uses 2**N.

src/shrpx.cc

@@ -365,12 +365,17 @@ void fill_default_config()
   // Timeout for pooled (idle) connections
   mod_config()->downstream_idle_read_timeout.tv_sec = 60;
 
-  // window bits for HTTP/2.0 and SPDY upstream/downstream
-  // connection. 2**16-1 = 64KiB-1, which is HTTP/2.0 default. Please
+  // window bits for HTTP/2.0 and SPDY upstream/downstream connection
+  // per stream. 2**16-1 = 64KiB-1, which is HTTP/2.0 default. Please
   // note that SPDY/3 default is 64KiB.
   mod_config()->http2_upstream_window_bits = 16;
   mod_config()->http2_downstream_window_bits = 16;
 
+  // HTTP/2.0 SPDY/3.1 has connection-level flow control. The default
+  // window size for HTTP/2 is 64KiB - 1. SPDY/3's default is 64KiB
+  mod_config()->http2_upstream_connection_window_bits = 16;
+  mod_config()->http2_downstream_connection_window_bits = 16;
+
   mod_config()->upstream_no_tls = false;
   mod_config()->downstream_no_tls = false;
 
@@ -620,16 +625,28 @@ void print_help(std::ostream& out)
       << "                       Default: "
       << get_config()->http2_max_concurrent_streams << "\n"
       << "    --frontend-http2-window-bits=<N>\n"
-      << "                       Sets the initial window size of HTTP/2.0 and SPDY\n"
-      << "                       frontend connection to 2**<N>-1.\n"
+      << "                       Sets the per-stream initial window size of HTTP/2.0\n"
+      << "                       SPDY frontend connection. For HTTP/2.0, the size is\n"
+      << "                       2**<N>-1. For SPDY, the size is 2**<N>\n"
       << "                       Default: "
       << get_config()->http2_upstream_window_bits << "\n"
+      << "    --frontend-http2-connection-window-bits=<N>\n"
+      << "                       Sets the per-connection window size of HTTP/2.0 and\n"
+      << "                       SPDY frontend connection. For HTTP/2.0, the size is\n"
+      << "                       2**<N>-1. For SPDY, the size is 2**<N>.\n"
+      << "                       Default: "
+      << get_config()->http2_upstream_connection_window_bits << "\n"
       << "    --frontend-no-tls  Disable SSL/TLS on frontend connections.\n"
       << "    --backend-http2-window-bits=<N>\n"
-      << "                       Sets the initial window size of HTTP/2.0 and SPDY\n"
-      << "                       backend connection to 2**<N>-1.\n"
+      << "                       Sets the initial window size of HTTP/2.0 backend\n"
+      << "                       connection to 2**<N>-1.\n"
       << "                       Default: "
       << get_config()->http2_downstream_window_bits << "\n"
+      << "    --backend-http2-connection-window-bits=<N>\n"
+      << "                       Sets the per-connection window size of HTTP/2.0\n"
+      << "                       backend connection to 2**<N>-1.\n"
+      << "                       Default: "
+      << get_config()->http2_downstream_connection_window_bits << "\n"
       << "    --backend-no-tls   Disable SSL/TLS on backend connections.\n"
       << "    --http2-no-cookie-crumbling\n"
       << "                       Don't crumble cookie header field.\n"
@@ -774,6 +791,8 @@ int main(int argc, char **argv)
       {"frontend-http2-dump-request-header", required_argument, &flag, 43},
       {"frontend-http2-dump-response-header", required_argument, &flag, 44},
       {"http2-no-cookie-crumbling", no_argument, &flag, 45},
+      {"frontend-http2-connection-window-bits", required_argument, &flag, 46},
+      {"backend-http2-connection-window-bits", required_argument, &flag, 47},
       {nullptr, 0, nullptr, 0 }
     };
 
@@ -1013,6 +1032,18 @@ int main(int argc, char **argv)
         cmdcfgs.push_back(std::make_pair
                           (SHRPX_OPT_HTTP2_NO_COOKIE_CRUMBLING, "yes"));
         break;
+      case 46:
+        // --frontend-http2-connection-window-bits
+        cmdcfgs.push_back(std::make_pair
+                          (SHRPX_OPT_FRONTEND_HTTP2_CONNECTION_WINDOW_BITS,
+                           optarg));
+        break;
+      case 47:
+        // --backend-http2-connection-window-bits
+        cmdcfgs.push_back(std::make_pair
+                          (SHRPX_OPT_BACKEND_HTTP2_CONNECTION_WINDOW_BITS,
+                           optarg));
+        break;
       default:
         break;
       }

src/shrpx_config.cc

@@ -80,6 +80,10 @@ SHRPX_OPT_BACKEND_KEEP_ALIVE_TIMEOUT[] = "backend-keep-alive-timeout";
 const char
 SHRPX_OPT_FRONTEND_HTTP2_WINDOW_BITS[] = "frontend-http2-window-bits";
 const char SHRPX_OPT_BACKEND_HTTP2_WINDOW_BITS[] = "backend-http2-window-bits";
+const char SHRPX_OPT_FRONTEND_HTTP2_CONNECTION_WINDOW_BITS[] =
+  "frontend-http2-connection-window-bits";
+const char SHRPX_OPT_BACKEND_HTTP2_CONNECTION_WINDOW_BITS[] =
+  "backend-http2-connection-window-bits";
 const char SHRPX_OPT_FRONTEND_NO_TLS[] = "frontend-no-tls";
 const char SHRPX_OPT_BACKEND_NO_TLS[] = "backend-no-tls";
 const char SHRPX_OPT_BACKEND_TLS_SNI_FIELD[] = "backend-tls-sni-field";
@@ -321,6 +325,28 @@ int parse_config(const char *opt, const char *optarg)
                  << " specify the integer in the range [0, 30], inclusive";
       return -1;
     }
+  } else if(util::strieq(opt,
+                         SHRPX_OPT_FRONTEND_HTTP2_CONNECTION_WINDOW_BITS) ||
+            util::strieq(opt,
+                         SHRPX_OPT_BACKEND_HTTP2_CONNECTION_WINDOW_BITS)) {
+    size_t *resp;
+    const char *optname;
+    if(util::strieq(opt, SHRPX_OPT_FRONTEND_HTTP2_CONNECTION_WINDOW_BITS)) {
+      resp = &mod_config()->http2_upstream_connection_window_bits;
+      optname = SHRPX_OPT_FRONTEND_HTTP2_CONNECTION_WINDOW_BITS;
+    } else {
+      resp = &mod_config()->http2_downstream_connection_window_bits;
+      optname = SHRPX_OPT_BACKEND_HTTP2_CONNECTION_WINDOW_BITS;
+    }
+    errno = 0;
+    unsigned long int n = strtoul(optarg, 0, 10);
+    if(errno == 0 && n >= 16 && n < 31) {
+      *resp = n;
+    } else {
+      LOG(ERROR) << "--" << optname
+                 << " specify the integer in the range [16, 30], inclusive";
+      return -1;
+    }
   } else if(util::strieq(opt, SHRPX_OPT_FRONTEND_NO_TLS)) {
     mod_config()->upstream_no_tls = util::strieq(optarg, "yes");
   } else if(util::strieq(opt, SHRPX_OPT_BACKEND_NO_TLS)) {

src/shrpx_config.h

@@ -71,6 +71,8 @@ extern const char SHRPX_OPT_ACCESSLOG[];
 extern const char SHRPX_OPT_BACKEND_KEEP_ALIVE_TIMEOUT[];
 extern const char SHRPX_OPT_FRONTEND_HTTP2_WINDOW_BITS[];
 extern const char SHRPX_OPT_BACKEND_HTTP2_WINDOW_BITS[];
+extern const char SHRPX_OPT_FRONTEND_HTTP2_CONNECTION_WINDOW_BITS[];
+extern const char SHRPX_OPT_BACKEND_HTTP2_CONNECTION_WINDOW_BITS[];
 extern const char SHRPX_OPT_FRONTEND_NO_TLS[];
 extern const char SHRPX_OPT_BACKEND_NO_TLS[];
 extern const char SHRPX_OPT_PID_FILE[];
@@ -146,6 +148,8 @@ struct Config {
   bool accesslog;
   size_t http2_upstream_window_bits;
   size_t http2_downstream_window_bits;
+  size_t http2_upstream_connection_window_bits;
+  size_t http2_downstream_connection_window_bits;
   bool upstream_no_tls;
   bool downstream_no_tls;
   char *backend_tls_sni_name;

src/shrpx_http2_session.cc

@@ -1170,6 +1170,16 @@ int Http2Session::on_connect()
     return -1;
   }
 
+  if(get_config()->http2_downstream_connection_window_bits > 16) {
+    int32_t delta =
+      (1 << get_config()->http2_downstream_connection_window_bits) - 1
+      - NGHTTP2_INITIAL_CONNECTION_WINDOW_SIZE;
+    rv = nghttp2_submit_window_update(session_, NGHTTP2_FLAG_NONE, 0, delta);
+    if(rv != 0) {
+      return -1;
+    }
+  }
+
   bufferevent_write(bev_, NGHTTP2_CLIENT_CONNECTION_HEADER,
                     NGHTTP2_CLIENT_CONNECTION_HEADER_LEN);
 

src/shrpx_http2_upstream.cc

@@ -513,6 +513,13 @@ Http2Upstream::Http2Upstream(ClientHandler *handler)
                                entry,
                                sizeof(entry)/sizeof(nghttp2_settings_entry));
   assert(rv == 0);
+
+  if(get_config()->http2_upstream_connection_window_bits > 16) {
+    int32_t delta = (1 << get_config()->http2_upstream_connection_window_bits)
+      - 1 - NGHTTP2_INITIAL_CONNECTION_WINDOW_SIZE;
+    rv = nghttp2_submit_window_update(session_, NGHTTP2_FLAG_NONE, 0, delta);
+    assert(rv == 0);
+  }
 }
 
 Http2Upstream::~Http2Upstream()

src/shrpx_spdy_upstream.cc

@@ -259,27 +259,27 @@ void on_data_chunk_recv_callback(spdylay_session *session,
       // returns 0.
       if(spdylay_session_get_recv_data_length(session) >
          std::max(SPDYLAY_INITIAL_WINDOW_SIZE,
-                  spdylay_session_get_local_window_size(session))) {
+                  1 << get_config()->http2_upstream_connection_window_bits)) {
         if(LOG_ENABLED(INFO)) {
           ULOG(INFO, upstream)
             << "Flow control error on connection: "
             << "recv_window_size="
             << spdylay_session_get_recv_data_length(session)
-            << ", initial_window_size="
-            << spdylay_session_get_local_window_size(session);
+            << ", window_size="
+            << (1 << get_config()->http2_upstream_connection_window_bits);
         }
         spdylay_session_fail_session(session, SPDYLAY_GOAWAY_PROTOCOL_ERROR);
         return;
       }
       if(spdylay_session_get_stream_recv_data_length(session, stream_id) >
          std::max(SPDYLAY_INITIAL_WINDOW_SIZE,
-                  spdylay_session_get_stream_local_window_size(session))) {
+                  1 << get_config()->http2_upstream_window_bits)) {
         if(LOG_ENABLED(INFO)) {
           ULOG(INFO, upstream)
             << "Flow control error: recv_window_size="
             << spdylay_session_get_stream_recv_data_length(session, stream_id)
             << ", initial_window_size="
-            << spdylay_session_get_stream_local_window_size(session);
+            << (1 << get_config()->http2_upstream_window_bits);
         }
         upstream->rst_stream(downstream, SPDYLAY_FLOW_CONTROL_ERROR);
         return;
@@ -395,10 +395,10 @@ SpdyUpstream::SpdyUpstream(uint16_t version, ClientHandler *handler)
   rv = spdylay_session_server_new(&session_, version, &callbacks, this);
   assert(rv == 0);
 
-  if(version == SPDYLAY_PROTO_SPDY3) {
+  if(version >= SPDYLAY_PROTO_SPDY3) {
     int val = 1;
     flow_control_ = true;
-    initial_window_size_ = (1 << get_config()->http2_upstream_window_bits) - 1;
+    initial_window_size_ = 1 << get_config()->http2_upstream_window_bits;
     rv = spdylay_session_set_option(session_,
                                     SPDYLAY_OPT_NO_AUTO_WINDOW_UPDATE, &val,
                                     sizeof(val));
@@ -421,6 +421,15 @@ SpdyUpstream::SpdyUpstream(uint16_t version, ClientHandler *handler)
     (session_, SPDYLAY_FLAG_SETTINGS_NONE,
      entry, sizeof(entry)/sizeof(spdylay_settings_entry));
   assert(rv == 0);
+
+  if(version >= SPDYLAY_PROTO_SPDY3_1 &&
+     get_config()->http2_upstream_connection_window_bits > 16) {
+    int32_t delta = (1 << get_config()->http2_upstream_connection_window_bits)
+      - SPDYLAY_INITIAL_WINDOW_SIZE;
+    rv = spdylay_submit_window_update(session_, 0, delta);
+    assert(rv == 0);
+  }
+
   // TODO Maybe call from outside?
   send();
 }
@@ -942,16 +951,14 @@ int32_t determine_window_update_transmission(spdylay_session *session,
   int32_t recv_length, window_size;
   if(stream_id == 0) {
     recv_length = spdylay_session_get_recv_data_length(session);
-    window_size = spdylay_session_get_local_window_size(session);
+    window_size = 1 << get_config()->http2_upstream_connection_window_bits;
   } else {
     recv_length = spdylay_session_get_stream_recv_data_length
       (session, stream_id);
-    window_size = spdylay_session_get_stream_local_window_size(session);
+    window_size = 1 << get_config()->http2_upstream_window_bits;
   }
-  if(recv_length != -1 && window_size != -1) {
-    if(recv_length >= window_size / 2) {
-      return recv_length;
-    }
+  if(recv_length != -1 && recv_length >= window_size / 2) {
+    return recv_length;
   }
   return -1;
 }