folly] hazptr: Add hazptr_cleanup function

Summary: The free function hazptr_cleanup reclaims all reclaimable objects retired to a domain before this call.

Reviewed By: djwatson

Differential Revision: D6876253

fbshipit-source-id: 54a6b98291712286597561e971fc9b6ac9c4d6f0

folly/experimental/hazptr/hazptr-impl.h

@@ -169,15 +169,147 @@ bool hazptr_tc_try_put(hazptr_rec* hprec);
 /** hazptr_priv structures
  *  Thread private lists of retired objects that belong to the default domain.
  */
-
-struct hazptr_priv {
-  hazptr_obj* head_;
-  hazptr_obj* tail_;
+class hazptr_priv {
+  std::atomic<hazptr_obj*> head_;
+  std::atomic<hazptr_obj*> tail_;
   int rcount_;
   bool active_;
+  hazptr_priv* prev_;
+  hazptr_priv* next_;
+
+ public:
+  void init() {
+    head_ = nullptr;
+    tail_ = nullptr;
+    rcount_ = 0;
+    active_ = true;
+  }
+
+  bool active() {
+    return active_;
+  }
+
+  hazptr_priv* prev() {
+    return prev_;
+  }
+
+  hazptr_priv* next() {
+    return next_;
+  }
+
+  bool empty() {
+    return head() == nullptr;
+  }
+
+  void set_prev(hazptr_priv* rec) {
+    prev_ = rec;
+  }
+
+  void set_next(hazptr_priv* rec) {
+    next_ = rec;
+  }
+
+  void clear_active() {
+    active_ = false;
+  }
+
+  void push(hazptr_obj* obj) {
+    while (true) {
+      if (tail()) {
+        if (pushInNonEmptyList(obj)) {
+          break;
+        }
+      } else {
+        if (pushInEmptyList(obj)) {
+          break;
+        }
+      }
+    }
+    if (++rcount_ >= HAZPTR_PRIV_THRESHOLD) {
+      push_all_to_domain();
+    }
+  }
+
+  void push_all_to_domain() {
+    auto& domain = default_hazptr_domain();
+    hazptr_obj* h = nullptr;
+    hazptr_obj* t = nullptr;
+    collect(h, t);
+    if (h) {
+      DCHECK(t);
+      domain.pushRetired(h, t, rcount_);
+    }
+    rcount_ = 0;
+    domain.tryBulkReclaim();
+  }
+
+  void collect(hazptr_obj*& colHead, hazptr_obj*& colTail) {
+    // This function doesn't change rcount_.
+    // The value rcount_ is accurate excluding the effects of collect().
+    auto h = exchangeHead();
+    if (h) {
+      auto t = exchangeTail();
+      DCHECK(t);
+      if (colTail) {
+        colTail->set_next(h);
+      } else {
+        colHead = h;
+      }
+      colTail = t;
+    }
+  }
+
+ private:
+  hazptr_obj* head() {
+    return head_.load(std::memory_order_acquire);
+  }
+
+  hazptr_obj* tail() {
+    return tail_.load(std::memory_order_acquire);
+  }
+
+  void setHead(hazptr_obj* obj) {
+    head_.store(obj, std::memory_order_release);
+  }
+
+  bool casHead(hazptr_obj* expected, hazptr_obj* obj) {
+    return head_.compare_exchange_weak(
+        expected, obj, std::memory_order_acq_rel, std::memory_order_relaxed);
+  }
 
-  void push(hazptr_obj* obj);
-  void pushAllToDomain();
+  bool casTail(hazptr_obj* expected, hazptr_obj* obj) {
+    return tail_.compare_exchange_weak(
+        expected, obj, std::memory_order_acq_rel, std::memory_order_relaxed);
+  }
+
+  hazptr_obj* exchangeHead() {
+    return head_.exchange(nullptr, std::memory_order_acq_rel);
+  }
+
+  hazptr_obj* exchangeTail() {
+    return tail_.exchange(nullptr, std::memory_order_acq_rel);
+  }
+
+  bool pushInNonEmptyList(hazptr_obj* obj) {
+    auto h = head();
+    if (h) {
+      obj->set_next(h);
+      if (casHead(h, obj)) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  bool pushInEmptyList(hazptr_obj* obj) {
+    hazptr_obj* t = nullptr;
+    obj->set_next(nullptr);
+    if (casTail(t, obj)) {
+      setHead(obj);
+      return true;
+    }
+    return false;
+  }
 };
 
 static_assert(
@@ -188,6 +320,49 @@ void hazptr_priv_init();
 void hazptr_priv_shutdown();
 bool hazptr_priv_try_retire(hazptr_obj* obj);
 
+inline void hazptr_priv_list::insert(hazptr_priv* rec) {
+  std::lock_guard<std::mutex> g(m_);
+  auto prev = head_ == nullptr ? rec : head_->prev();
+  auto next = head_ == nullptr ? rec : head_;
+  rec->set_next(next);
+  rec->set_prev(prev);
+  if (head_) {
+    prev->set_next(rec);
+    next->set_prev(rec);
+  } else {
+    head_ = rec;
+  }
+}
+
+inline void hazptr_priv_list::remove(hazptr_priv* rec) {
+  std::lock_guard<std::mutex> g(m_);
+  auto prev = rec->prev();
+  auto next = rec->next();
+  if (next == rec) {
+    DCHECK(prev == rec);
+    DCHECK(head_ == rec);
+    head_ = nullptr;
+  } else {
+    prev->set_next(next);
+    next->set_prev(prev);
+    if (head_ == rec) {
+      head_ = next;
+    }
+  }
+}
+
+inline void hazptr_priv_list::collect(hazptr_obj*& head, hazptr_obj*& tail) {
+  std::lock_guard<std::mutex> g(m_);
+  auto rec = head_;
+  while (rec) {
+    rec->collect(head, tail);
+    rec = rec->next();
+    if (rec == head_) {
+      break;
+    }
+  }
+}
+
 /** hazptr_tls_life */
 
 struct hazptr_tls_life {
@@ -648,6 +823,10 @@ FOLLY_ALWAYS_INLINE void hazptr_retire(T* obj, D reclaim) {
   default_hazptr_domain().retire(obj, std::move(reclaim));
 }
 
+inline void hazptr_cleanup(hazptr_domain& domain) {
+  domain.cleanup();
+}
+
 /** hazptr_rec */
 
 FOLLY_ALWAYS_INLINE void hazptr_rec::set(const void* p) noexcept {
@@ -740,6 +919,17 @@ inline hazptr_domain::~hazptr_domain() {
   }
 }
 
+inline void hazptr_domain::cleanup() {
+  hazptr_obj* h = nullptr;
+  hazptr_obj* t = nullptr;
+  priv_.collect(h, t);
+  if (h) {
+    DCHECK(t);
+    pushRetired(h, t, 0);
+  }
+  bulkReclaim();
+}
+
 inline hazptr_rec* hazptr_domain::hazptrAcquire() {
   hazptr_rec* p;
   hazptr_rec* next;
@@ -1025,49 +1215,19 @@ FOLLY_ALWAYS_INLINE bool hazptr_tc_try_put(hazptr_rec* hprec) {
  *  hazptr_priv
  */
 
-inline void hazptr_priv::push(hazptr_obj* obj) {
-  auto& domain = default_hazptr_domain();
-  obj->next_ = nullptr;
-  if (tail_) {
-    tail_->next_ = obj;
-  } else {
-    if (!active_) {
-      domain.objRetire(obj);
-      return;
-    }
-    head_ = obj;
-  }
-  tail_ = obj;
-  if (++rcount_ >= HAZPTR_PRIV_THRESHOLD) {
-    pushAllToDomain();
-  }
-}
-
-inline void hazptr_priv::pushAllToDomain() {
-  auto& domain = default_hazptr_domain();
-  domain.pushRetired(head_, tail_, rcount_);
-  head_ = nullptr;
-  tail_ = nullptr;
-  rcount_ = 0;
-  domain.tryBulkReclaim();
-}
-
 inline void hazptr_priv_init() {
   auto& priv = tls_priv_data_;
   HAZPTR_DEBUG_PRINT(&priv);
-  priv.head_ = nullptr;
-  priv.tail_ = nullptr;
-  priv.rcount_ = 0;
-  priv.active_ = true;
+  priv.init();
 }
 
 inline void hazptr_priv_shutdown() {
   auto& priv = tls_priv_data_;
   HAZPTR_DEBUG_PRINT(&priv);
-  DCHECK(priv.active_);
-  priv.active_ = false;
-  if (priv.tail_) {
-    priv.pushAllToDomain();
+  DCHECK(priv.active());
+  priv.clear_active();
+  if (!priv.empty()) {
+    priv.push_all_to_domain();
   }
 }
 

folly/experimental/hazptr/hazptr.h

@@ -17,6 +17,7 @@
 #define HAZPTR_H
 
 #include <atomic>
+#include <mutex>
 
 /* Stand-in for C++17 std::pmr::memory_resource */
 #include <folly/experimental/hazptr/memory_resource.h>
@@ -49,9 +50,32 @@ class hazptr_array;
 template <size_t M>
 class hazptr_local;
 
+/** hazptr_priv: Per-thread list of retired objects pushed in bulk to domain */
+class hazptr_priv;
+
+class hazptr_priv_list {
+  std::mutex m_;
+  hazptr_priv* head_{nullptr};
+
+ public:
+  void insert(hazptr_priv* rec);
+  void remove(hazptr_priv* rec);
+  void collect(hazptr_obj*& head, hazptr_obj*& tail);
+};
+
 /** hazptr_domain: Class of hazard pointer domains. Each domain manages a set
  *  of hazard pointers and a set of retired objects. */
 class hazptr_domain {
+  memory_resource* mr_;
+  std::atomic<hazptr_rec*> hazptrs_ = {nullptr};
+  std::atomic<hazptr_obj*> retired_ = {nullptr};
+  /* Using signed int for rcount_ because it may transiently be
+   * negative.  Using signed int for all integer variables that may be
+   * involved in calculations related to the value of rcount_. */
+  std::atomic<int> hcount_ = {0};
+  std::atomic<int> rcount_ = {0};
+  hazptr_priv_list priv_;
+
  public:
   constexpr explicit hazptr_domain(
       memory_resource* = get_default_resource()) noexcept;
@@ -65,6 +89,7 @@ class hazptr_domain {
   /** Free-function retire.  May allocate memory */
   template <typename T, typename D = std::default_delete<T>>
   void retire(T* obj, D reclaim = {});
+  void cleanup();
 
  private:
   friend class hazptr_obj_batch;
@@ -73,16 +98,7 @@ class hazptr_domain {
   friend class hazptr_obj_base;
   template <typename, typename>
   friend class hazptr_obj_base_refcounted;
-  friend struct hazptr_priv;
-
-  memory_resource* mr_;
-  std::atomic<hazptr_rec*> hazptrs_ = {nullptr};
-  std::atomic<hazptr_obj*> retired_ = {nullptr};
-  std::atomic<int> hcount_ = {0};
-  std::atomic<int> rcount_ = {0};
-  /* Using signed int for rcount_ because it may transiently be
-   * negative.  Using signed int for all integer variables that may be
-   * involved in calculations related to the value of rcount_. */
+  friend class hazptr_priv;
 
   void objRetire(hazptr_obj*);
   hazptr_rec* hazptrAcquire();
@@ -102,6 +118,11 @@ extern hazptr_domain default_domain_;
 template <typename T, typename D = std::default_delete<T>>
 void hazptr_retire(T* obj, D reclaim = {});
 
+/** hazptr_cleanup
+ *  Reclaims all reclaimable objects retired to the domain before this call.
+ */
+void hazptr_cleanup(hazptr_domain& domain = default_hazptr_domain());
+
 /** Definition of hazptr_obj */
 class hazptr_obj {
   friend class hazptr_obj_batch;
@@ -110,7 +131,7 @@ class hazptr_obj {
   friend class hazptr_obj_base;
   template <typename, typename>
   friend class hazptr_obj_base_refcounted;
-  friend struct hazptr_priv;
+  friend class hazptr_priv;
 
   void (*reclaim_)(hazptr_obj*);
   hazptr_obj* next_;
@@ -122,6 +143,10 @@ class hazptr_obj {
   }
 
  private:
+  void set_next(hazptr_obj* obj) {
+    next_ = obj;
+  }
+
   void retireCheck() {
     // Only for catching misuse bugs like double retire
     if (next_ != this) {

folly/experimental/hazptr/test/HazptrTest.cpp

@@ -575,3 +575,24 @@ TEST_F(HazptrTest, FreeFunctionRetire) {
   }
   EXPECT_TRUE(retired);
 }
+
+TEST_F(HazptrTest, FreeFunctionCleanup) {
+  CHECK_GT(FLAGS_num_threads, 0);
+  constructed.store(0);
+  destroyed.store(0);
+  std::vector<std::thread> threads(FLAGS_num_threads);
+  for (int tid = 0; tid < FLAGS_num_threads; ++tid) {
+    threads[tid] = std::thread([&, tid]() {
+      for (int j = tid; j < FLAGS_num_ops; j += FLAGS_num_threads) {
+        auto p = new Foo(j, nullptr);
+        p->retire();
+      }
+    });
+  }
+  for (auto& t : threads) {
+    t.join();
+  }
+  CHECK_EQ(constructed.load(), FLAGS_num_ops);
+  hazptr_cleanup();
+  CHECK_EQ(destroyed.load(), FLAGS_num_ops);
+}