Commit 93232f95 authored by Yedidya Feldblum's avatar Yedidya Feldblum Committed by Facebook Github Bot

Use atomic operations on TLS SharedMutex static data

Summary:
[Folly] Use atomic operations on maybe-TLS `SharedMutex` static data  via `atomic_ref`. The fields are thread-local on platforms which support thread-local fields, but global otherwise.

When thread-local is supported, relaxed loads and stores should ordinarily have the same overhead as non-atomic loads and stores on most platforms. When thread-local is not supported and these fields are global, the concurrent atomic loads and stores will change to become defined behavior.

Differential Revision: D19688867

fbshipit-source-id: 51873e12924ecff9d971316658fe71e2de18bdec
parent 0a24425a
...@@ -30,6 +30,7 @@ ...@@ -30,6 +30,7 @@
#include <folly/detail/Futex.h> #include <folly/detail/Futex.h>
#include <folly/portability/Asm.h> #include <folly/portability/Asm.h>
#include <folly/portability/SysResource.h> #include <folly/portability/SysResource.h>
#include <folly/synchronization/AtomicRef.h>
#include <folly/synchronization/SanitizeThread.h> #include <folly/synchronization/SanitizeThread.h>
// SharedMutex is a reader-writer lock. It is small, very fast, scalable // SharedMutex is a reader-writer lock. It is small, very fast, scalable
...@@ -1605,13 +1606,15 @@ bool SharedMutexImpl< ...@@ -1605,13 +1606,15 @@ bool SharedMutexImpl<
Atom, Atom,
BlockImmediately, BlockImmediately,
AnnotateForThreadSanitizer>::tryUnlockTokenlessSharedDeferred() { AnnotateForThreadSanitizer>::tryUnlockTokenlessSharedDeferred() {
auto bestSlot = tls_lastTokenlessSlot; auto bestSlot =
make_atomic_ref(tls_lastTokenlessSlot).load(std::memory_order_relaxed);
for (uint32_t i = 0; i < kMaxDeferredReaders; ++i) { for (uint32_t i = 0; i < kMaxDeferredReaders; ++i) {
auto slotPtr = deferredReader(bestSlot ^ i); auto slotPtr = deferredReader(bestSlot ^ i);
auto slotValue = slotPtr->load(std::memory_order_relaxed); auto slotValue = slotPtr->load(std::memory_order_relaxed);
if (slotValue == tokenlessSlotValue() && if (slotValue == tokenlessSlotValue() &&
slotPtr->compare_exchange_strong(slotValue, 0)) { slotPtr->compare_exchange_strong(slotValue, 0)) {
tls_lastTokenlessSlot = bestSlot ^ i; make_atomic_ref(tls_lastTokenlessSlot)
.store(bestSlot ^ i, std::memory_order_relaxed);
return true; return true;
} }
} }
...@@ -1638,7 +1641,8 @@ bool SharedMutexImpl< ...@@ -1638,7 +1641,8 @@ bool SharedMutexImpl<
return false; return false;
} }
uint32_t slot = tls_lastDeferredReaderSlot; uint32_t slot = make_atomic_ref(tls_lastDeferredReaderSlot)
.load(std::memory_order_relaxed);
uintptr_t slotValue = 1; // any non-zero value will do uintptr_t slotValue = 1; // any non-zero value will do
bool canAlreadyDefer = (state & kMayDefer) != 0; bool canAlreadyDefer = (state & kMayDefer) != 0;
...@@ -1662,7 +1666,8 @@ bool SharedMutexImpl< ...@@ -1662,7 +1666,8 @@ bool SharedMutexImpl<
slotValue = deferredReader(slot)->load(std::memory_order_relaxed); slotValue = deferredReader(slot)->load(std::memory_order_relaxed);
if (slotValue == 0) { if (slotValue == 0) {
// found empty slot // found empty slot
tls_lastDeferredReaderSlot = slot; make_atomic_ref(tls_lastDeferredReaderSlot)
.store(slot, std::memory_order_relaxed);
break; break;
} }
} }
...@@ -1713,7 +1718,8 @@ bool SharedMutexImpl< ...@@ -1713,7 +1718,8 @@ bool SharedMutexImpl<
} }
if (token == nullptr) { if (token == nullptr) {
tls_lastTokenlessSlot = slot; make_atomic_ref(tls_lastTokenlessSlot)
.store(slot, std::memory_order_relaxed);
} }
if ((state & kMayDefer) != 0) { if ((state & kMayDefer) != 0) {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment