Skip to content

F
folly
Commit 82f9889c authored by Maged Michael's avatar Maged Michael Committed by Facebook GitHub Bot
Browse files
Options

hazard pointers: Consolidate non-cohort and untagged cohort retired objects 

Summary:
Consolidate retired_, the list of non-cohort retired objects, with untagged_, the list of cohort untagged objects, in the domain.

Eliminate redundant functions.

Reviewed By: davidtgoldblatt

Differential Revision: D30806882

fbshipit-source-id: bdcbc4990a3d114f7f6bea294df498c5743bed43
parent b3833ef1
Hide whitespace changes
Inline Side-by-side
Showing with 11 additions and 171 deletions
folly/synchronization/Hazptr-fwd.h
View file @ 82f9889c
......@@ -142,17 +142,10 @@ hazptr_domain<Atom>& default_hazptr_domain();
template <template <typename> class Atom = std::atomic>
hazard_pointer_domain<Atom>& hazard_pointer_default_domain();
/** hazptr_domain_push_list */
template <template <typename> class Atom = std::atomic>
void hazptr_domain_push_list(
hazptr_obj_list<Atom>& l,
hazptr_domain<Atom>& domain = default_hazptr_domain<Atom>()) noexcept;
/** hazptr_domain_push_retired */
template <template <typename> class Atom = std::atomic>
void hazptr_domain_push_retired(
hazptr_obj_list<Atom>& l,
bool check = true,
hazptr_domain<Atom>& domain = default_hazptr_domain<Atom>()) noexcept;
/** hazptr_retire */
......
folly/synchronization/HazptrDomain.h
View file @ 82f9889c
......@@ -117,13 +117,11 @@ class hazptr_domain {
}
Atom<hazptr_rec<Atom>*> hazptrs_{nullptr};
Atom<hazptr_obj<Atom>*> retired_{nullptr};
Atom<uint64_t> sync_time_{0};
/* Using signed int for rcount_ because it may transiently be negative.
Using signed int for all integer variables that may be involved in
calculations related to the value of rcount_. */
Atom<int> hcount_{0};
Atom<int> rcount_{0};
Atom<uint16_t> num_bulk_reclaims_{0};
bool shutdown_{false};
RetiredList untagged_;
......@@ -174,12 +172,11 @@ class hazptr_domain {
delete static_cast<hazptr_retire_node*>(p);
};
hazptr_obj_list<Atom> l(node);
push_retired(l);
push_list(l);
}
/** cleanup */
void cleanup() noexcept {
relaxed_cleanup();
inc_num_bulk_reclaims();
do_reclamation(0);
wait_for_zero_bulk_reclaims(); // wait for concurrent bulk_reclaim-s
......@@ -214,10 +211,8 @@ class hazptr_domain {
hazptr_rec<Atom>,
FixedAlign<alignof(hazptr_rec<Atom>)>>;
friend void hazptr_domain_push_list<Atom>(
hazptr_obj_list<Atom>&, hazptr_domain<Atom>&) noexcept;
friend void hazptr_domain_push_retired<Atom>(
hazptr_obj_list<Atom>&, bool check, hazptr_domain<Atom>&) noexcept;
hazptr_obj_list<Atom>&, hazptr_domain<Atom>&) noexcept;
friend hazptr_holder<Atom> make_hazard_pointer<Atom>(hazptr_domain<Atom>&);
friend class hazptr_holder<Atom>;
friend class hazptr_obj<Atom>;
......@@ -275,26 +270,6 @@ class hazptr_domain {
/** hprec_release */
void hprec_release(hazptr_rec<Atom>* hprec) noexcept { hprec->release(); }
/** push_retired */
void push_retired(hazptr_obj_list<Atom>& l, bool check = true) {
/*** Full fence ***/ asymmetricLightBarrier();
while (true) {
auto r = retired();
l.tail()->set_next(r);
if (retired_.compare_exchange_weak(
r,
l.head(),
std::memory_order_release,
std::memory_order_acquire)) {
break;
}
}
rcount_.fetch_add(l.count(), std::memory_order_release);
if (check) {
check_cleanup_and_reclaim();
}
}
/** push_list */
void push_list(ObjList& l) {
if (l.empty()) {
......@@ -438,7 +413,7 @@ class hazptr_domain {
}
/** match_reclaim_untagged */
int match_reclaim_untagged(Obj* untagged, Set& hs) {
int match_reclaim_untagged(Obj* untagged, Set& hs, bool& done) {
ObjList match, nomatch;
list_match_condition(untagged, match, nomatch, [&](Obj* o) {
return hs.count(o->raw_ptr()) > 0;
......@@ -446,6 +421,7 @@ class hazptr_domain {
ObjList children;
int count = nomatch.count();
reclaim_unprotected(nomatch.head(), children);
done = untagged_.empty() && children.empty();
count -= children.count();
match.splice(children);
List l(match.head(), match.tail());
......@@ -458,36 +434,23 @@ class hazptr_domain {
while (true) {
Obj* untagged;
Obj* tagged[kNumShards];
bool done = true;
if (extract_retired_objects(untagged, tagged)) {
/*** Full fence ***/ asymmetricHeavyBarrier(AMBFlags::EXPEDITED);
Set hs = load_hazptr_vals();
rcount -= match_tagged(tagged, hs);
rcount -= match_reclaim_untagged(untagged, hs);
rcount -= match_reclaim_untagged(untagged, hs, done);
}
if (rcount) {
add_count(rcount);
}
rcount = check_count_threshold();
if (rcount == 0)
if (rcount == 0 && done)
break;
}
dec_num_bulk_reclaims();
}
/** lookup_and_reclaim */
void lookup_and_reclaim(Obj* obj, const Set& hs, ObjList& keep) {
while (obj) {
auto next = obj->next();
DCHECK_NE(obj, next);
if (hs.count(obj->raw_ptr()) == 0) {
(*(obj->reclaim()))(obj, keep);
} else {
keep.push(obj);
}
obj = next;
}
}
/** list_match_condition */
template <typename Cond>
void list_match_condition(
......@@ -526,26 +489,12 @@ class hazptr_domain {
return hazptrs_.load(std::memory_order_acquire);
}
hazptr_obj<Atom>* retired() const noexcept {
return retired_.load(std::memory_order_acquire);
}
int hcount() const noexcept {
return hcount_.load(std::memory_order_acquire);
}
int rcount() const noexcept {
return rcount_.load(std::memory_order_acquire);
}
bool reached_threshold(int rc, int hc) const noexcept {
return rc >= kThreshold && rc >= kMultiplier * hc;
}
void reclaim_all_objects() {
auto head = retired_.exchange(nullptr);
reclaim_list_transitive(head);
head = untagged_.pop_all(RetiredList::kDontLock);
Obj* head = untagged_.pop_all(RetiredList::kDontLock);
reclaim_list_transitive(head);
}
......@@ -573,103 +522,12 @@ class hazptr_domain {
}
}
void check_cleanup_and_reclaim() {
if (try_timed_cleanup()) {
return;
}
if (reached_threshold(rcount(), hcount())) {
try_bulk_reclaim();
}
}
void relaxed_cleanup() noexcept {
rcount_.store(0, std::memory_order_release);
bulk_reclaim(true);
}
void wait_for_zero_bulk_reclaims() {
while (load_num_bulk_reclaims() > 0) {
std::this_thread::yield();
}
}
void try_bulk_reclaim() {
auto hc = hcount();
auto rc = rcount();
if (!reached_threshold(rc, hc)) {
return;
}
rc = rcount_.exchange(0, std::memory_order_release);
if (!reached_threshold(rc, hc)) {
/* No need to add rc back to rcount_. At least one concurrent
try_bulk_reclaim will proceed to bulk_reclaim. */
return;
}
bulk_reclaim();
}
void bulk_reclaim(bool transitive = false) {
inc_num_bulk_reclaims();
while (true) {
auto obj = retired_.exchange(nullptr, std::memory_order_acquire);
/*** Full fence ***/ asymmetricHeavyBarrier(AMBFlags::EXPEDITED);
auto rec = hazptrs_.load(std::memory_order_acquire);
/* Part 1 - read hazard pointer values into private search structure */
std::unordered_set<const void*> hashset;
for (; rec; rec = rec->next()) {
hashset.insert(rec->hazptr());
}
/* Part 2 - for each retired object, reclaim if no match */
if (bulk_lookup_and_reclaim(obj, hashset) || !transitive) {
break;
}
}
dec_num_bulk_reclaims();
}
bool bulk_lookup_and_reclaim(
hazptr_obj<Atom>* obj, const std::unordered_set<const void*>& hashset) {
hazptr_obj_list<Atom> children;
hazptr_obj_list<Atom> matched;
while (obj) {
auto next = obj->next();
DCHECK_NE(obj, next);
if (hashset.count(obj->raw_ptr()) == 0) {
(*(obj->reclaim()))(obj, children);
} else {
matched.push(obj);
}
obj = next;
}
bool done = ((children.count() == 0) && (retired() == nullptr));
matched.splice(children);
if (matched.count() > 0) {
push_retired(matched, false /* don't call bulk_reclaim recursively */);
}
return done;
}
bool check_sync_time(Atom<uint64_t>& sync_time) {
uint64_t time = std::chrono::duration_cast<std::chrono::nanoseconds>(
std::chrono::steady_clock::now().time_since_epoch())
.count();
auto prevtime = sync_time.load(std::memory_order_relaxed);
if (time < prevtime ||
!sync_time.compare_exchange_strong(
prevtime, time + kSyncTimePeriod, std::memory_order_relaxed)) {
return false;
}
return true;
}
bool try_timed_cleanup() {
if (!check_sync_time(sync_time_)) {
return false;
}
relaxed_cleanup(); // calling regular cleanup may self deadlock
return true;
}
hazptr_rec<Atom>* try_acquire_existing_hprec() {
auto rec = head();
while (rec) {
......@@ -791,15 +649,6 @@ hazard_pointer_default_domain() {
/** hazptr_domain_push_retired: push a list of retired objects into a domain */
template <template <typename> class Atom>
void hazptr_domain_push_retired(
hazptr_obj_list<Atom>& l,
bool check,
hazptr_domain<Atom>& domain) noexcept {
domain.push_retired(l, check);
}
/** hazptr_domain_push_list */
template <template <typename> class Atom>
void hazptr_domain_push_list(
hazptr_obj_list<Atom>& l, hazptr_domain<Atom>& domain) noexcept {
domain.push_list(l);
}
......
folly/synchronization/HazptrObj.h
View file @ 82f9889c
......@@ -146,8 +146,6 @@ class hazptr_obj {
friend class hazptr_domain<Atom>;
template <typename, template <typename> class, typename>
friend class hazptr_obj_base;
template <typename, template <typename> class, typename>
friend class hazptr_obj_base_refcounted;
friend class hazptr_obj_cohort<Atom>;
Obj* next() const noexcept { return next_; }
......@@ -177,7 +175,7 @@ class hazptr_obj {
void push_to_retired(hazptr_domain<Atom>& domain) {
hazptr_obj_list<Atom> l(this);
hazptr_domain_push_retired(l, true, domain);
hazptr_domain_push_retired(l, domain);
}
FOLLY_NOINLINE void pre_retire_check_fail() noexcept {
......@@ -368,7 +366,7 @@ class hazptr_obj_cohort {
}
if (!children.empty()) {
if (active()) {
hazptr_domain_push_list<Atom>(children);
hazptr_domain_push_retired<Atom>(children);
} else {
obj = children.head();
}
......@@ -393,7 +391,7 @@ class hazptr_obj_cohort {
}
}
hazptr_obj_list<Atom> l(ll.head(), ll.tail(), c);
hazptr_domain_push_list<Atom>(l);
hazptr_domain_push_retired<Atom>(l);
return;
}
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment