Add deadlock detector to folly and thrift server

Summary:
- Add DeadlockDetector interface to folly/concurrency
- Add IOThreadPoolDeadlockDetectorObserver class that can be registered in IOThreadPoolExecutor to enable the functionality for its threads that use EventBase.
- In ThriftServer use the new observer with IOThreadPoolExecutor
- Default implementation of deadlock detector is stubbed out and does not do anything

Reviewed By: mshneer

Differential Revision: D28582490

fbshipit-source-id: 1cf3c2957b46de83f07c53bab708fe09ebbe2b79

folly/concurrency/DeadlockDetector.cpp

+/*
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <folly/concurrency/DeadlockDetector.h>
+
+namespace folly {
+/* static */ DeadlockDetectorFactory* DeadlockDetectorFactory::instance() {
+  if (get_deadlock_detector_factory_instance) {
+    return get_deadlock_detector_factory_instance();
+  }
+
+  return nullptr;
+}
+} // namespace folly

folly/concurrency/DeadlockDetector.h

+/*
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <folly/Executor.h>
+
+namespace folly {
+class DeadlockDetector {
+ public:
+  virtual ~DeadlockDetector() {}
+};
+
+class DeadlockDetectorFactory {
+ public:
+  virtual ~DeadlockDetectorFactory() {}
+  virtual std::unique_ptr<DeadlockDetector> create(
+      Executor* executor, const std::string& name) = 0;
+  static DeadlockDetectorFactory* instance();
+};
+
+using GetDeadlockDetectorFactoryInstance = DeadlockDetectorFactory*();
+#if FOLLY_HAVE_WEAK_SYMBOLS
+FOLLY_ATTR_WEAK GetDeadlockDetectorFactoryInstance
+    get_deadlock_detector_factory_instance;
+#else
+constexpr GetDeadlockDetectorFactoryInstance*
+    get_deadlock_detector_factory_instance = nullptr;
+#endif
+} // namespace folly

folly/executors/IOThreadPoolDeadlockDetectorObserver.cpp

+/*
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <memory>
+#include <folly/concurrency/DeadlockDetector.h>
+#include <folly/executors/IOThreadPoolDeadlockDetectorObserver.h>
+
+namespace folly {
+
+IOThreadPoolDeadlockDetectorObserver::IOThreadPoolDeadlockDetectorObserver(
+    DeadlockDetectorFactory* deadlockDetectorFactory, const std::string& name)
+    : name_(name), deadlockDetectorFactory_(deadlockDetectorFactory) {}
+
+void IOThreadPoolDeadlockDetectorObserver::threadStarted(
+    folly::ThreadPoolExecutor::ThreadHandle* h) {
+  if (!deadlockDetectorFactory_) {
+    return;
+  }
+
+  auto eventBase = folly::IOThreadPoolExecutor::getEventBase(h);
+  // This Observer only works with IOThreadPoolExecutor class.
+  CHECK_NOTNULL(eventBase);
+  detectors_.wlock()->insert_or_assign(
+      h, deadlockDetectorFactory_->create(eventBase, name_));
+}
+
+void IOThreadPoolDeadlockDetectorObserver::threadStopped(
+    folly::ThreadPoolExecutor::ThreadHandle* h) {
+  if (!deadlockDetectorFactory_) {
+    return;
+  }
+
+  detectors_.wlock()->erase(h);
+}
+
+/* static */ std::unique_ptr<IOThreadPoolDeadlockDetectorObserver>
+IOThreadPoolDeadlockDetectorObserver::create(const std::string& name) {
+  auto* deadlockDetectorFactory = DeadlockDetectorFactory::instance();
+  return std::make_unique<IOThreadPoolDeadlockDetectorObserver>(
+      deadlockDetectorFactory, name);
+}
+
+} // namespace folly

folly/executors/IOThreadPoolDeadlockDetectorObserver.h

+/*
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <folly/Singleton.h>
+#include <folly/concurrency/DeadlockDetector.h>
+#include <folly/executors/IOThreadPoolExecutor.h>
+#include <folly/executors/ThreadPoolExecutor.h>
+
+namespace folly {
+
+class IOThreadPoolDeadlockDetectorObserver
+    : public folly::ThreadPoolExecutor::Observer {
+ public:
+  IOThreadPoolDeadlockDetectorObserver(
+      folly::DeadlockDetectorFactory* deadlockDetectorFactory,
+      const std::string& name);
+  void threadStarted(folly::ThreadPoolExecutor::ThreadHandle* h) override;
+  void threadStopped(folly::ThreadPoolExecutor::ThreadHandle* h) override;
+
+  static std::unique_ptr<IOThreadPoolDeadlockDetectorObserver> create(
+      const std::string& name);
+
+ private:
+  const std::string name_;
+  folly::DeadlockDetectorFactory* deadlockDetectorFactory_;
+  folly::Synchronized<std::unordered_map<
+      folly::ThreadPoolExecutor::ThreadHandle*,
+      std::unique_ptr<folly::DeadlockDetector>>>
+      detectors_;
+};
+
+} // namespace folly

folly/executors/test/IOThreadPoolDeadlockDetectorObserverTest.cpp

+/*
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <memory>
+#include <folly/concurrency/DeadlockDetector.h>
+#include <folly/executors/IOThreadPoolDeadlockDetectorObserver.h>
+#include <folly/executors/IOThreadPoolExecutor.h>
+#include <folly/portability/GTest.h>
+
+using namespace folly;
+using namespace testing;
+
+class DeadlockDetectorMock : public DeadlockDetector {
+ public:
+  DeadlockDetectorMock(std::shared_ptr<std::atomic<int32_t>> counter)
+      : counter_(counter) {
+    counter_->fetch_add(1);
+  }
+  ~DeadlockDetectorMock() override { counter_->fetch_sub(1); }
+
+ private:
+  std::shared_ptr<std::atomic<int32_t>> counter_;
+};
+
+class DeadlockDetectorFactoryMock : public DeadlockDetectorFactory {
+ public:
+  std::unique_ptr<DeadlockDetector> create(
+      folly::Executor* executor, const std::string& name) override {
+    EXPECT_TRUE(executor != nullptr);
+    EXPECT_EQ("TestPool", name);
+    return std::make_unique<DeadlockDetectorMock>(counter_);
+  }
+
+  int32_t getCounter() { return counter_->load(); }
+
+ private:
+  std::shared_ptr<std::atomic<int32_t>> counter_ =
+      std::make_shared<std::atomic<int32_t>>(0);
+};
+
+TEST(
+    IOThreadPoolDeadlockDetectorObserverTest,
+    VerifyDeadlockDetectorCreatedAndDestroyedOnAddRemoveObserver) {
+  auto deadlockDetectorFactory =
+      std::make_shared<DeadlockDetectorFactoryMock>();
+
+  auto executor = std::make_shared<IOThreadPoolExecutor>(
+      1, std::make_shared<folly::NamedThreadFactory>("TestPool"));
+
+  auto observer = std::make_shared<folly::IOThreadPoolDeadlockDetectorObserver>(
+      deadlockDetectorFactory.get(), "TestPool");
+
+  ASSERT_EQ(0, deadlockDetectorFactory->getCounter())
+      << "Deadlock detector not yet registered";
+
+  executor->addObserver(observer);
+  ASSERT_EQ(1, deadlockDetectorFactory->getCounter())
+      << "Deadlock detector must be registered by observer";
+
+  executor->removeObserver(observer);
+  ASSERT_EQ(0, deadlockDetectorFactory->getCounter())
+      << "Removing observer must destroy deadlock detector";
+
+  executor->addObserver(observer);
+  ASSERT_EQ(1, deadlockDetectorFactory->getCounter())
+      << "Deadlock detector must be registered by observer";
+
+  executor->stop();
+  ASSERT_EQ(0, deadlockDetectorFactory->getCounter())
+      << "Stopping threadpool must deregister all observers";
+}