Erasing Cpp2Worker in its evb thread

Summary: Erasing Cpp2Worker happens in acceptor thread but when evb is
destroyed and drains its callback queue, the callbacks may access the
worker. This moves erasing Cpp2Worer to its own evb thread, after all
callbacks are run.

Reviewed By: @yfeldblum

Differential Revision: D2398089

folly/io/async/EventBase.cpp

@@ -238,6 +238,12 @@ EventBase::~EventBase() {
     event_base_free(evb_);
   }
 
+  while (!runAfterDrainCallbacks_.empty()) {
+    LoopCallback* callback = &runAfterDrainCallbacks_.front();
+    runAfterDrainCallbacks_.pop_front();
+    callback->runLoopCallback();
+  }
+
   {
     std::lock_guard<std::mutex> lock(localStorageMutex_);
     for (auto storage : localStorageToDtor_) {
@@ -526,6 +532,13 @@ void EventBase::runInLoop(Cob&& cob, bool thisIteration) {
   }
 }
 
+void EventBase::runAfterDrain(Cob&& cob) {
+  auto callback = new FunctionLoopCallback<Cob>(std::move(cob));
+  std::lock_guard<std::mutex> lg(runAfterDrainCallbacksMutex_);
+  callback->cancelLoopCallback();
+  runAfterDrainCallbacks_.push_back(*callback);
+}
+
 void EventBase::runOnDestruction(LoopCallback* callback) {
   std::lock_guard<std::mutex> lg(onDestructionCallbacksMutex_);
   callback->cancelLoopCallback();

folly/io/async/EventBase.h

@@ -317,6 +317,15 @@ class EventBase : private boost::noncopyable,
    */
   void runOnDestruction(LoopCallback* callback);
 
+  /**
+   * Adds the given callback to a queue of things run after the notification
+   * queue is drained before the destruction of current EventBase.
+   *
+   * Note: will be called from the thread that invoked EventBase destructor,
+   *       after the final run of loop callbacks.
+   */
+  void runAfterDrain(Cob&& cob);
+
   /**
    * Adds a callback that will run immediately *before* the event loop.
    * This is very similar to runInLoop(), but will not cause the loop to break:
@@ -677,6 +686,7 @@ class EventBase : private boost::noncopyable,
   LoopCallbackList loopCallbacks_;
   LoopCallbackList runBeforeLoopCallbacks_;
   LoopCallbackList onDestructionCallbacks_;
+  LoopCallbackList runAfterDrainCallbacks_;
 
   // This will be null most of the time, but point to currentCallbacks
   // if we are in the middle of running loop callbacks, such that
@@ -744,6 +754,9 @@ class EventBase : private boost::noncopyable,
   // allow runOnDestruction() to be called from any threads
   std::mutex onDestructionCallbacksMutex_;
 
+  // allow runAfterDrain() to be called from any threads
+  std::mutex runAfterDrainCallbacksMutex_;
+
   // see EventBaseLocal
   friend class detail::EventBaseLocalBase;
   template <typename T> friend class EventBaseLocal;