Hazard pointers: Add support for thread local lists of retired objects and other optimizations

Summary:
- Add support for private lists of retired objects
- Add an option for one domain
- More scalable thread cache managemnt
- hazptr_rec alignment
- FOLLY_ALWAYS_INLINE
- Refactor management of retired objects in hazptr_domain
- Refactor benchmarks

Reviewed By: davidtgoldblatt

Differential Revision: D5322646

fbshipit-source-id: 9d31582b9a8216861e7e78e2e1cd08dc11cf7f88

folly/experimental/hazptr/bench/HazptrBench-Amb-NoTc.cpp

@@ -16,6 +16,7 @@
 
 #define HAZPTR_AMB true
 #define HAZPTR_TC false
+#define HAZPTR_PRIV false
 
 #include <folly/experimental/hazptr/bench/HazptrBench.h>
 #include <folly/portability/GFlags.h>
@@ -23,29 +24,8 @@
 
 using namespace folly::hazptr;
 
-int nthreads;
-int size;
-
-BENCHMARK(amb, iters) {
-  run_once(nthreads, size, iters);
-}
-
-BENCHMARK(amb_dup, iters) {
-  run_once(nthreads, size, iters);
-}
-
 int main(int argc, char** argv) {
   testing::InitGoogleTest(&argc, argv);
   gflags::ParseCommandLineFlags(&argc, &argv, true);
-  std::cout << "---------------------------------------------- AMB - No TC\n";
-  for (int i : nthr) {
-    nthreads = i;
-    for (int j : sizes) {
-      size = j;
-      std::cout << i << " threads -- " << j << "-item list" << std::endl;
-      bench("amb - no tc                 ", i, j, 0);
-      bench("amb - no tc - dup           ", i, j, 0);
-    }
-  }
-  std::cout << "----------------------------------------------------------\n";
+  benches("   amb - no tc");
 }

folly/experimental/hazptr/bench/HazptrBench-Amb-Tc.cpp

@@ -16,6 +16,7 @@
 
 #define HAZPTR_AMB true
 #define HAZPTR_TC true
+#define HAZPTR_PRIV true
 
 #include <folly/experimental/hazptr/bench/HazptrBench.h>
 #include <folly/portability/GFlags.h>
@@ -23,29 +24,8 @@
 
 using namespace folly::hazptr;
 
-int nthreads;
-int size;
-
-BENCHMARK(amb, iters) {
-  run_once(nthreads, size, iters);
-}
-
-BENCHMARK(amb_dup, iters) {
-  run_once(nthreads, size, iters);
-}
-
 int main(int argc, char** argv) {
   testing::InitGoogleTest(&argc, argv);
   gflags::ParseCommandLineFlags(&argc, &argv, true);
-  std::cout << "------------------------------------------------- AMB - TC\n";
-  for (int i : nthr) {
-    nthreads = i;
-    for (int j : sizes) {
-      size = j;
-      std::cout << i << " threads -- " << j << "-item list" << std::endl;
-      bench("amb - tc                    ", i, j, 0);
-      bench("amb - tc - dup              ", i, j, 0);
-    }
-  }
-  std::cout << "----------------------------------------------------------\n";
+  benches("   amb -    tc");
 }

folly/experimental/hazptr/bench/HazptrBench-NoAmb-NoTc.cpp

@@ -16,6 +16,7 @@
 
 #define HAZPTR_AMB false
 #define HAZPTR_TC false
+#define HAZPTR_PRIV false
 
 #include <folly/experimental/hazptr/bench/HazptrBench.h>
 #include <folly/portability/GFlags.h>
@@ -23,29 +24,8 @@
 
 using namespace folly::hazptr;
 
-int nthreads;
-int size;
-
-BENCHMARK(no_amb, iters) {
-  run_once(nthreads, size, iters);
-}
-
-BENCHMARK(no_amb_dup, iters) {
-  run_once(nthreads, size, iters);
-}
-
 int main(int argc, char** argv) {
   testing::InitGoogleTest(&argc, argv);
   gflags::ParseCommandLineFlags(&argc, &argv, true);
-  std::cout << "------------------------------------------- No AMB - No Tc\n";
-  for (int i : nthr) {
-    nthreads = i;
-    for (int j : sizes) {
-      size = j;
-      std::cout << i << " threads -- " << j << "-item list" << std::endl;
-      bench("no amb - no tc              ", i, j, 0);
-      bench("no amb - no tc - dup        ", i, j, 0);
-    }
-  }
-  std::cout << "----------------------------------------------------------\n";
+  benches("no amb - no tc");
 }

folly/experimental/hazptr/bench/HazptrBench-NoAmb-Tc.cpp

@@ -16,6 +16,7 @@
 
 #define HAZPTR_AMB false
 #define HAZPTR_TC true
+#define HAZPTR_PRIV true
 
 #include <folly/experimental/hazptr/bench/HazptrBench.h>
 #include <folly/portability/GFlags.h>
@@ -23,29 +24,8 @@
 
 using namespace folly::hazptr;
 
-int nthreads;
-int size;
-
-BENCHMARK(no_amb, iters) {
-  run_once(nthreads, size, iters);
-}
-
-BENCHMARK(no_amb_dup, iters) {
-  run_once(nthreads, size, iters);
-}
-
 int main(int argc, char** argv) {
   testing::InitGoogleTest(&argc, argv);
   gflags::ParseCommandLineFlags(&argc, &argv, true);
-  std::cout << "---------------------------------------------- No AMB - TC\n";
-  for (int i : nthr) {
-    nthreads = i;
-    for (int j : sizes) {
-      size = j;
-      std::cout << i << " threads -- " << j << "-item list" << std::endl;
-      bench("no amb - tc                 ", i, j, 0);
-      bench("no amb - tc - dup           ", i, j, 0);
-    }
-  }
-  std::cout << "----------------------------------------------------------\n";
+  benches("no amb -    tc");
 }

folly/experimental/hazptr/bench/HazptrBench-OneDomain.cpp

+/*
+ * Copyright 2017 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define HAZPTR_AMB true
+#define HAZPTR_TC true
+#define HAZPTR_PRIV true
+#define HAZPTR_ONE_DOMAIN true
+
+#include <folly/experimental/hazptr/bench/HazptrBench.h>
+#include <folly/portability/GFlags.h>
+#include <folly/portability/GTest.h>
+
+using namespace folly::hazptr;
+
+int main(int argc, char** argv) {
+  testing::InitGoogleTest(&argc, argv);
+  gflags::ParseCommandLineFlags(&argc, &argv, true);
+  benches("    one domain");
+}

folly/experimental/hazptr/hazptr.h

@@ -17,9 +17,6 @@
 #define HAZPTR_H
 
 #include <atomic>
-#include <functional>
-#include <memory>
-#include <type_traits>
 
 /* Stand-in for C++17 std::pmr::memory_resource */
 #include <folly/experimental/hazptr/memory_resource.h>
@@ -51,9 +48,10 @@ class hazptr_domain {
   hazptr_domain& operator=(hazptr_domain&&) = delete;
 
  private:
+  friend class hazptr_holder;
   template <typename, typename>
   friend class hazptr_obj_base;
-  friend class hazptr_holder;
+  friend class hazptr_priv;
 
   memory_resource* mr_;
   std::atomic<hazptr_rec*> hazptrs_ = {nullptr};
@@ -65,6 +63,7 @@ class hazptr_domain {
   hazptr_rec* hazptrAcquire();
   void hazptrRelease(hazptr_rec*) noexcept;
   int pushRetired(hazptr_obj* head, hazptr_obj* tail, int count);
+  bool reachedThreshold(int rcount);
   void tryBulkReclaim();
   void bulkReclaim();
 };
@@ -77,6 +76,7 @@ class hazptr_obj {
   friend class hazptr_domain;
   template <typename, typename>
   friend class hazptr_obj_base;
+  friend class hazptr_priv;
 
   void (*reclaim_)(hazptr_obj*);
   hazptr_obj* next_;
@@ -84,17 +84,15 @@ class hazptr_obj {
 };
 
 /** Definition of hazptr_obj_base */
-template <typename T, typename Deleter = std::default_delete<T>>
+template <typename T, typename D = std::default_delete<T>>
 class hazptr_obj_base : public hazptr_obj {
  public:
   /* Retire a removed object and pass the responsibility for
    * reclaiming it to the hazptr library */
-  void retire(
-      hazptr_domain& domain = default_hazptr_domain(),
-      Deleter reclaim = {});
+  void retire(hazptr_domain& domain = default_hazptr_domain(), D reclaim = {});
 
  private:
-  Deleter deleter_;
+  D deleter_;
 };
 
 /** hazptr_holder: Class for automatic acquisition and release of

folly/experimental/hazptr/memory_resource.h

@@ -15,6 +15,8 @@
  */
 #pragma once
 
+#include <folly/experimental/hazptr/debug.h>
+
 ////////////////////////////////////////////////////////////////////////////////
 /// Disclaimer: This is intended only as a partial stand-in for
 /// std::pmr::memory_resource (C++17) as needed for developing a
@@ -45,7 +47,6 @@ memory_resource* new_delete_resource();
 ////////////////////////////////////////////////////////////////////////////////
 /// Implementation
 ////////////////////////////////////////////////////////////////////////////////
-#include <folly/experimental/hazptr/debug.h>
 
 inline memory_resource** default_mr_ptr() {
   /* library-local */ static memory_resource* default_mr =