Crash process if folly::Singleton destruction is taking too long

Summary: This provides a single mechanism to mitigate deadlocks in singleton destructors.

Reviewed By: yfeldblum

Differential Revision: D21827615

fbshipit-source-id: 090bad1a91238310e7814663193711df0f4d622a

CMake/folly-config.h.cmake

@@ -82,3 +82,5 @@
 #cmakedefine01 FOLLY_LIBRARY_SANITIZE_ADDRESS
 
 #cmakedefine FOLLY_SUPPORT_SHARED_LIBRARY 1
+
+#cmakedefine01 FOLLY_HAVE_LIBRT

folly/Singleton-inl.h

@@ -181,7 +181,9 @@ void SingletonHolder<T>::destroyInstance() {
     auto last_reference_released =
         destroy_baton_->try_wait_for(kDestroyWaitTime, wait_options);
     if (last_reference_released) {
+      vault_.addToShutdownLog("Destroying " + type().name());
       teardown_(instance_ptr_);
+      vault_.addToShutdownLog(type().name() + " destroyed.");
     } else {
       print_destructor_stack_trace_->store(true);
       detail::singletonWarnDestroyInstanceLeak(type(), instance_ptr_);

folly/Singleton.cpp

@@ -15,10 +15,11 @@
  */
 
 #include <folly/Singleton.h>
-#include <folly/portability/Config.h>
 
 #ifndef _WIN32
 #include <dlfcn.h>
+#include <signal.h>
+#include <time.h>
 #endif
 
 #include <atomic>
@@ -31,6 +32,7 @@
 #include <folly/Format.h>
 #include <folly/ScopeGuard.h>
 #include <folly/detail/SingletonStackTrace.h>
+#include <folly/portability/Config.h>
 
 #if !defined(_WIN32) && !defined(__APPLE__) && !defined(__ANDROID__)
 #define FOLLY_SINGLETON_HAVE_DLSYM 1
@@ -38,6 +40,8 @@
 
 namespace folly {
 
+constexpr std::chrono::seconds SingletonVault::kDefaultShutdownTimeout_;
+
 #if FOLLY_SINGLETON_HAVE_DLSYM
 namespace detail {
 static void singleton_hs_init_weak(int* argc, char** argv[])
@@ -365,7 +369,58 @@ void SingletonVault::scheduleDestroyInstances() {
   // Add a dependency on folly::ThreadLocal to make sure all its static
   // singletons are initalized first.
   threadlocal_detail::StaticMeta<void, void>::instance();
-  std::atexit([] { SingletonVault::singleton()->destroyInstances(); });
+  std::atexit([] {
+    SingletonVault::singleton()->startShutdownTimer();
+    SingletonVault::singleton()->destroyInstances();
+  });
+}
+
+void SingletonVault::addToShutdownLog(std::string message) {
+  shutdownLog_.wlock()->push_back(std::move(message));
+}
+
+#if FOLLY_HAVE_LIBRT
+namespace {
+[[noreturn]] void fireShutdownSignalHelper(sigval_t sigval) {
+  static_cast<SingletonVault*>(sigval.sival_ptr)->fireShutdownTimer();
+}
+} // namespace
+#endif
+
+void SingletonVault::startShutdownTimer() {
+#if FOLLY_HAVE_LIBRT
+  if (shutdownTimerStarted_.exchange(true)) {
+    return;
+  }
+
+  struct sigevent sig;
+  sig.sigev_notify = SIGEV_THREAD;
+  sig.sigev_notify_function = fireShutdownSignalHelper;
+  sig.sigev_value.sival_ptr = this;
+  sig.sigev_notify_attributes = nullptr;
+  timer_t timerId;
+  PCHECK(timer_create(CLOCK_MONOTONIC, &sig, &timerId) == 0);
+
+  struct itimerspec newValue, oldValue;
+  newValue.it_value.tv_sec =
+      std::chrono::milliseconds(shutdownTimeout_).count() / 1000;
+  newValue.it_value.tv_nsec =
+      std::chrono::milliseconds(shutdownTimeout_).count() % 1000 * 1000000;
+  newValue.it_interval.tv_sec = 0;
+  newValue.it_interval.tv_nsec = 0;
+  PCHECK(timer_settime(timerId, 0, &newValue, &oldValue) == 0);
+#endif
+}
+
+[[noreturn]] void SingletonVault::fireShutdownTimer() {
+  std::string shutdownLog;
+  for (auto& logMessage : shutdownLog_.copy()) {
+    shutdownLog += logMessage + "\n";
+  }
+  LOG(FATAL) << "Failed to complete shutdown within "
+             << std::chrono::milliseconds(shutdownTimeout_).count()
+             << "ms. Shutdown log:\n"
+             << shutdownLog;
 }
 
 } // namespace folly

folly/Singleton.h

@@ -516,6 +516,16 @@ class SingletonVault {
     type_ = type;
   }
 
+  void setShutdownTimeout(std::chrono::milliseconds shutdownTimeout) {
+    shutdownTimeout_ = shutdownTimeout;
+  }
+
+  void addToShutdownLog(std::string message);
+
+  void startShutdownTimer();
+
+  [[noreturn]] void fireShutdownTimer();
+
  private:
   template <typename T>
   friend struct detail::SingletonHolder;
@@ -556,6 +566,12 @@ class SingletonVault {
   Synchronized<detail::SingletonVaultState, SharedMutexReadPriority> state_;
 
   Type type_;
+
+  std::atomic<bool> shutdownTimerStarted_{false};
+  static constexpr std::chrono::seconds kDefaultShutdownTimeout_{
+      std::chrono::seconds{5 * 60}};
+  std::chrono::milliseconds shutdownTimeout_{kDefaultShutdownTimeout_};
+  Synchronized<std::vector<std::string>> shutdownLog_;
 };
 
 // This is the wrapper class that most users actually interact with.

folly/init/Init.cpp

@@ -95,6 +95,7 @@ Init::Init(int* argc, char*** argv, InitOptions options) {
 }
 
 Init::~Init() {
+  SingletonVault::singleton()->startShutdownTimer();
   SingletonVault::singleton()->destroyInstances();
 }
 } // namespace folly

folly/test/SingletonTest.cpp

@@ -33,6 +33,7 @@
 FOLLY_GNU_DISABLE_WARNING("-Wdeprecated-declarations")
 
 using namespace folly;
+using namespace std::chrono_literals;
 
 TEST(Singleton, MissingSingleton) {
   EXPECT_DEATH(
@@ -1021,3 +1022,34 @@ TEST(Singleton, LeakySingletonLSAN) {
   EXPECT_NE(ptr0, ptr1);
   EXPECT_EQ(*ptr1, 1);
 }
+
+TEST(Singleton, ShutdownTimer) {
+  struct VaultTag {};
+  struct PrivateTag {};
+  struct Object {
+    ~Object() {
+      /* sleep override */ std::this_thread::sleep_for(shutdownDuration);
+    }
+
+    std::chrono::milliseconds shutdownDuration;
+  };
+  using SingletonObject = Singleton<Object, PrivateTag, VaultTag>;
+
+  auto& vault = *SingletonVault::singleton<VaultTag>();
+  SingletonObject object;
+  vault.registrationComplete();
+
+  vault.setShutdownTimeout(10ms);
+  SingletonObject::try_get()->shutdownDuration = 1s;
+  EXPECT_DEATH(
+      [&]() {
+        vault.startShutdownTimer();
+        vault.destroyInstances();
+      }(),
+      "Failed to complete shutdown within 10ms.");
+
+  vault.setShutdownTimeout(1s);
+  SingletonObject::try_get()->shutdownDuration = 10ms;
+  vault.startShutdownTimer();
+  vault.destroyInstances();
+}