hazptr: Move reclamation outside tagged list critical section

Summary:
Move the reclamation of tagged objects outside the critical section on the tagged list lock (in asynchronous reclamation of the domain tagged list).

Tagged objects are subject to both synchronous and asynchronous reclamation. Each tagged object belongs to a cohort.

Currently, asynchronous reclamation of the domain list of tagged objects reclaims tagged objects while holding the lock on the list, so that synchronous reclamation of a cohort's objects does not miss such objects.

This diff adds a singly linked list of objects that are safe to reclaim to the cohort structure. Asynchronous reclamation pushes reclaimable tagged objects into the safe lists in their respective cohorts, instead of reclaiming such objects while holding the lock on the list of tagged objects. By pushing objects to their respective cohorts, synchronous reclamation can find not-yet-reclaimed objects in the cohort's own safe list.

Reviewed By: davidtgoldblatt

Differential Revision: D23973107

fbshipit-source-id: 2a9758cc8b03bb98a45b5579bab2964d9ae3d60c

folly/synchronization/HazptrDomain.h

@@ -352,15 +352,21 @@ class hazptr_domain {
     list_match_condition(obj, match, nomatch, [&](Obj* o) {
       return hs.count(o->raw_ptr()) > 0;
     });
-    /* Reclaim unprotected objects and push back protected objects and
-       children of reclaimed objects */
     if (lock) {
-      unprotected_ = nomatch.head();
-      DCHECK(children_.empty());
-      reclaim_unprotected_safe();
-      match.splice(children_);
+      /* Push unprotected objects into their cohorts and push protected
+         objects back into the list and unlock it */
+      obj = nomatch.head();
+      while (obj) {
+        auto next = obj->next();
+        auto cohort = obj->cohort();
+        DCHECK(cohort);
+        cohort->push_safe_obj(obj);
+        obj = next;
+      }
       rlist.push_unlock(match);
     } else {
+      /* Reclaim unprotected objects and push protected objects and
+         children of reclaimed objects into the list */
       ObjList children;
       reclaim_unprotected_unsafe(nomatch.head(), children);
       match.splice(children);

folly/synchronization/HazptrObj.h

@@ -300,11 +300,16 @@ class hazptr_obj_cohort {
   Atom<int> count_;
   Atom<bool> active_;
   Atom<bool> pushed_to_domain_tagged_;
+  Atom<Obj*> safe_list_top_;
 
  public:
   /** Constructor */
   hazptr_obj_cohort() noexcept
-      : l_(), count_(0), active_(true), pushed_to_domain_tagged_{false} {}
+      : l_(),
+        count_(0),
+        active_(true),
+        pushed_to_domain_tagged_{false},
+        safe_list_top_{nullptr} {}
 
   /** Not copyable or moveable */
   hazptr_obj_cohort(const hazptr_obj_cohort& o) = delete;
@@ -328,6 +333,7 @@ class hazptr_obj_cohort {
     if (pushed_to_domain_tagged_.load(std::memory_order_relaxed)) {
       default_hazptr_domain<Atom>().cleanup_cohort_tag(this);
     }
+    reclaim_safe_list();
     if (!l_.empty()) {
       List l = l_.pop_all();
       clear_count();
@@ -338,6 +344,7 @@ class hazptr_obj_cohort {
   }
 
  private:
+  friend class hazptr_domain<Atom>;
   friend class hazptr_obj<Atom>;
 
   bool active() {
@@ -365,18 +372,39 @@ class hazptr_obj_cohort {
         expected, newval, std::memory_order_acq_rel, std::memory_order_acquire);
   }
 
+  Obj* safe_list_top() const noexcept {
+    return safe_list_top_.load(std::memory_order_acquire);
+  }
+
+  bool cas_safe_list_top(Obj*& expected, Obj* newval) noexcept {
+    return safe_list_top_.compare_exchange_weak(
+        expected, newval, std::memory_order_acq_rel, std::memory_order_relaxed);
+  }
+
   /** push_obj */
   void push_obj(Obj* obj) {
     if (active()) {
       l_.push(obj);
       inc_count();
       check_threshold_push();
+      if (safe_list_top())
+        reclaim_safe_list();
     } else {
       obj->set_next(nullptr);
       reclaim_list(obj);
     }
   }
 
+  /** push_safe_obj */
+  void push_safe_obj(Obj* obj) noexcept {
+    while (true) {
+      Obj* top = safe_list_top();
+      obj->set_next(top);
+      if (cas_safe_list_top(top, obj))
+        return;
+    }
+  }
+
   /** reclaim_list */
   void reclaim_list(hazptr_obj<Atom>* obj) {
     while (obj) {
@@ -412,6 +440,12 @@ class hazptr_obj_cohort {
       }
     }
   }
+
+  /** reclaim_safe_list */
+  void reclaim_safe_list() {
+    Obj* top = safe_list_top_.exchange(nullptr, std::memory_order_acq_rel);
+    reclaim_list(top);
+  }
 }; // hazptr_obj_cohort
 
 /**