nghttpx: Don't stop default loop of worker process on graceful shutdown

To keep ipc channel being read from worker process, default loop
should not be stopped.  To join all worker threads, now we use
dedicated thread to do this.  When all worker threads are joined,
ev_async_send sends message to default loop, and it is finally
stopped.

src/shrpx_connection_handler.cc

@@ -96,6 +96,12 @@ void ocsp_chld_cb(struct ev_loop *loop, ev_child *w, int revent) {
 }
 } // namespace
 
+namespace {
+void thread_join_async_cb(struct ev_loop *loop, ev_async *w, int revent) {
+  ev_break(loop);
+}
+} // namespace
+
 ConnectionHandler::ConnectionHandler(struct ev_loop *loop)
     : single_worker_(nullptr), loop_(loop),
       tls_ticket_key_memcached_get_retry_count_(0),
@@ -110,6 +116,8 @@ ConnectionHandler::ConnectionHandler(struct ev_loop *loop)
   ev_io_init(&ocsp_.rev, ocsp_read_cb, -1, EV_READ);
   ocsp_.rev.data = this;
 
+  ev_async_init(&thread_join_asyncev_, thread_join_async_cb);
+
   ev_child_init(&ocsp_.chldev, ocsp_chld_cb, 0, 0);
   ocsp_.chldev.data = this;
 
@@ -120,6 +128,7 @@ ConnectionHandler::ConnectionHandler(struct ev_loop *loop)
 }
 
 ConnectionHandler::~ConnectionHandler() {
+  ev_async_stop(loop_, &thread_join_asyncev_);
   ev_timer_stop(loop_, &disable_acceptor_timer_);
   ev_timer_stop(loop_, &ocsp_timer_);
 
@@ -264,9 +273,19 @@ void ConnectionHandler::graceful_shutdown_worker() {
   }
 
   for (auto &worker : workers_) {
-
     worker->send(wev);
   }
+
+#ifndef NOTHREADS
+  ev_async_start(loop_, &thread_join_asyncev_);
+
+  thread_join_fut_ = std::async(std::launch::async, [this]() {
+    (void)reopen_log_files();
+    join_worker();
+    ev_async_send(get_loop(), &thread_join_asyncev_);
+    delete log_config();
+  });
+#endif // NOTHREADS
 }
 
 int ConnectionHandler::handle_connection(int fd, sockaddr *addr, int addrlen) {

src/shrpx_connection_handler.h

@@ -34,6 +34,9 @@
 
 #include <memory>
 #include <vector>
+#ifndef NOTHREADS
+#include <future>
+#endif // NOTHREADS
 
 #include <openssl/ssl.h>
 
@@ -158,6 +161,10 @@ private:
 #endif // HAVE_NEVERBLEED
   ev_timer disable_acceptor_timer_;
   ev_timer ocsp_timer_;
+  ev_async thread_join_asyncev_;
+#ifndef NOTHREADS
+  std::future<void> thread_join_fut_;
+#endif // NOTHREADS
   size_t tls_ticket_key_memcached_get_retry_count_;
   size_t tls_ticket_key_memcached_fail_count_;
   unsigned int worker_round_robin_cnt_;

src/shrpx_worker_process.cc

@@ -112,19 +112,14 @@ void graceful_shutdown(ConnectionHandler *conn_handler) {
 
   conn_handler->graceful_shutdown_worker();
 
-  if (get_config()->num_worker == 1 &&
-      conn_handler->get_single_worker()->get_worker_stat()->num_connections >
-          0) {
+  if (get_config()->num_worker == 1) {
+    if (conn_handler->get_single_worker()->get_worker_stat()->num_connections ==
+        0) {
+      ev_break(conn_handler->get_loop());
+    }
+
     return;
   }
-
-  // We have accepted all pending connections.  Shutdown main event
-  // loop.
-
-  // TODO this makes IPC from master process impossible.  Perhaps, we
-  // should keep alive default loop, and break it once all connections
-  // are terminated somehow.
-  ev_break(conn_handler->get_loop());
 }
 } // namespace
 
@@ -527,7 +522,6 @@ int worker_process_event_loop(WorkerProcessConfig *wpconf) {
 
   ev_run(loop, 0);
 
-  conn_handler.join_worker();
   conn_handler.cancel_ocsp_update();
 
 #ifdef HAVE_NEVERBLEED