Reduce memory usage of CoreCachedSharedPtr

Summary:
We only need as many slots as the number of L1 caches.

Also avoid allocating control blocks when the passed pointer has no managed object.

Reviewed By: philippv, luciang

Differential Revision: D29872059

fbshipit-source-id: 8c221b0523494c44a5c6828bafd26eeb00e573c4

folly/concurrency/CacheLocality.h

@@ -484,8 +484,9 @@ class CoreRawAllocator {
 };
 
 template <typename T, size_t Stripes>
-CxxAllocatorAdaptor<T, typename CoreRawAllocator<Stripes>::Allocator>
-getCoreAllocator(size_t stripe) {
+FOLLY_EXPORT
+    CxxAllocatorAdaptor<T, typename CoreRawAllocator<Stripes>::Allocator>
+    getCoreAllocator(size_t stripe) {
   // We cannot make sure that the allocator will be destroyed after
   // all the objects allocated with it, so we leak it.
   static Indestructible<CoreRawAllocator<Stripes>> allocator;

folly/concurrency/CoreCachedSharedPtr.h

@@ -17,18 +17,56 @@
 #pragma once
 
 #include <array>
+#include <atomic>
 #include <memory>
 
+#include <folly/CppAttributes.h>
 #include <folly/Portability.h>
+#include <folly/Unit.h>
 #include <folly/concurrency/CacheLocality.h>
-#include <folly/container/Enumerate.h>
 #include <folly/synchronization/Hazptr.h>
 
 namespace folly {
 
 // On mobile we do not expect high concurrency, and memory is more important, so
 // use more conservative caching.
-constexpr size_t kCoreCachedSharedPtrDefaultNumSlots = kIsMobile ? 4 : 64;
+constexpr size_t kCoreCachedSharedPtrDefaultMaxSlots = kIsMobile ? 4 : 64;
+
+namespace core_cached_shared_ptr_detail {
+
+template <size_t kMaxSlots>
+class SlotsConfig {
+ public:
+  FOLLY_EXPORT static void initialize() {
+    FOLLY_MAYBE_UNUSED static const Unit _ = [] {
+      // We need at most as many slots as the number of L1 caches, so we can
+      // avoid wasting memory if more slots are requested.
+      const auto l1Caches = CacheLocality::system().numCachesByLevel.front();
+      num_ = std::min(std::max<size_t>(1, l1Caches), kMaxSlots);
+      return unit;
+    }();
+  }
+
+  static size_t num() { return num_.load(std::memory_order_relaxed); }
+
+ private:
+  static std::atomic<size_t> num_;
+};
+
+// Initialize with a valid num so that get() always returns a valid stripe, even
+// if initialize() has not been called yet.
+template <size_t kMaxSlots>
+std::atomic<size_t> SlotsConfig<kMaxSlots>::num_{1};
+
+// Check whether a shared_ptr is equivalent to default-constructed. Because of
+// aliasing constructors, there can be both nullptr with a managed object, and
+// non-nullptr with no managed object, so we need to check both.
+template <class T>
+bool isDefault(const std::shared_ptr<T>& p) {
+  return p == nullptr && p.use_count() == 0;
+}
+
+} // namespace core_cached_shared_ptr_detail
 
 /**
  * This class creates core-local caches for a given shared_ptr, to
@@ -36,29 +74,36 @@ constexpr size_t kCoreCachedSharedPtrDefaultNumSlots = kIsMobile ? 4 : 64;
  *
  * It has the same thread-safety guarantees as shared_ptr: it is safe
  * to concurrently call get(), but reset()s must be synchronized with
- * reads and other resets().
+ * reads and other reset()s.
  *
  * @author Giuseppe Ottaviano <ott@fb.com>
  */
-template <class T, size_t kNumSlots = kCoreCachedSharedPtrDefaultNumSlots>
+template <class T, size_t kMaxSlots = kCoreCachedSharedPtrDefaultMaxSlots>
 class CoreCachedSharedPtr {
+  using SlotsConfig = core_cached_shared_ptr_detail::SlotsConfig<kMaxSlots>;
+
  public:
   CoreCachedSharedPtr() = default;
   explicit CoreCachedSharedPtr(const std::shared_ptr<T>& p) { reset(p); }
 
   void reset(const std::shared_ptr<T>& p = nullptr) {
+    SlotsConfig::initialize();
     // Allocate each Holder in a different CoreRawAllocator stripe to
     // prevent false sharing. Their control blocks will be adjacent
     // thanks to allocate_shared().
-    for (auto slot : folly::enumerate(slots_)) {
-      auto alloc = getCoreAllocator<Holder, kNumSlots>(slot.index);
-      auto holder = std::allocate_shared<Holder>(alloc, p);
-      *slot = std::shared_ptr<T>(holder, p.get());
+    for (size_t i = 0; i < SlotsConfig::num(); ++i) {
+      // Try freeing the control block before allocating a new one.
+      slots_[i] = {};
+      if (!core_cached_shared_ptr_detail::isDefault(p)) {
+        auto alloc = getCoreAllocator<Holder, kMaxSlots>(i);
+        auto holder = std::allocate_shared<Holder>(alloc, p);
+        slots_[i] = std::shared_ptr<T>(holder, p.get());
+      }
     }
   }
 
   std::shared_ptr<T> get() const {
-    return slots_[AccessSpreader<>::cachedCurrent(kNumSlots)];
+    return slots_[AccessSpreader<>::cachedCurrent(SlotsConfig::num())];
   }
 
  private:
@@ -67,35 +112,38 @@ class CoreCachedSharedPtr {
   template <class, size_t>
   friend class CoreCachedWeakPtr;
 
-  std::array<std::shared_ptr<T>, kNumSlots> slots_;
+  std::array<std::shared_ptr<T>, kMaxSlots> slots_;
 };
 
-template <class T, size_t kNumSlots = kCoreCachedSharedPtrDefaultNumSlots>
+template <class T, size_t kMaxSlots = kCoreCachedSharedPtrDefaultMaxSlots>
 class CoreCachedWeakPtr {
+  using SlotsConfig = core_cached_shared_ptr_detail::SlotsConfig<kMaxSlots>;
+
  public:
   CoreCachedWeakPtr() = default;
-  explicit CoreCachedWeakPtr(const CoreCachedSharedPtr<T, kNumSlots>& p) {
+  explicit CoreCachedWeakPtr(const CoreCachedSharedPtr<T, kMaxSlots>& p) {
     reset(p);
   }
 
   void reset() { *this = {}; }
-  void reset(const CoreCachedSharedPtr<T, kNumSlots>& p) {
-    for (auto slot : folly::enumerate(slots_)) {
-      *slot = p.slots_[slot.index];
+  void reset(const CoreCachedSharedPtr<T, kMaxSlots>& p) {
+    SlotsConfig::initialize();
+    for (size_t i = 0; i < SlotsConfig::num(); ++i) {
+      slots_[i] = p.slots_[i];
     }
   }
 
   std::weak_ptr<T> get() const {
-    return slots_[AccessSpreader<>::cachedCurrent(kNumSlots)];
+    return slots_[AccessSpreader<>::cachedCurrent(SlotsConfig::num())];
   }
 
   // Faster than get().lock(), as it avoid one weak count cycle.
   std::shared_ptr<T> lock() const {
-    return slots_[AccessSpreader<>::cachedCurrent(kNumSlots)].lock();
+    return slots_[AccessSpreader<>::cachedCurrent(SlotsConfig::num())].lock();
   }
 
  private:
-  std::array<std::weak_ptr<T>, kNumSlots> slots_;
+  std::array<std::weak_ptr<T>, kMaxSlots> slots_;
 };
 
 /**
@@ -110,52 +158,53 @@ class CoreCachedWeakPtr {
  * get()s will never see a newer pointer on one core, and an older
  * pointer on another after a subsequent thread migration.
  */
-template <class T, size_t kNumSlots = kCoreCachedSharedPtrDefaultNumSlots>
+template <class T, size_t kMaxSlots = kCoreCachedSharedPtrDefaultMaxSlots>
 class AtomicCoreCachedSharedPtr {
+  using SlotsConfig = core_cached_shared_ptr_detail::SlotsConfig<kMaxSlots>;
+
  public:
-  explicit AtomicCoreCachedSharedPtr(const std::shared_ptr<T>& p = nullptr) {
-    reset(p);
-  }
+  AtomicCoreCachedSharedPtr() = default;
+  explicit AtomicCoreCachedSharedPtr(const std::shared_ptr<T>& p) { reset(p); }
 
   ~AtomicCoreCachedSharedPtr() {
-    auto slots = slots_.load(std::memory_order_acquire);
     // Delete of AtomicCoreCachedSharedPtr must be synchronized, no
     // need for slots->retire().
-    if (slots) {
-      delete slots;
-    }
+    delete slots_.load(std::memory_order_acquire);
   }
 
   void reset(const std::shared_ptr<T>& p = nullptr) {
-    auto newslots = std::make_unique<Slots>();
-    // Allocate each Holder in a different CoreRawAllocator stripe to
-    // prevent false sharing. Their control blocks will be adjacent
-    // thanks to allocate_shared().
-    for (auto slot : folly::enumerate(newslots->slots_)) {
-      auto alloc = getCoreAllocator<Holder, kNumSlots>(slot.index);
-      auto holder = std::allocate_shared<Holder>(alloc, p);
-      *slot = std::shared_ptr<T>(holder, p.get());
+    SlotsConfig::initialize();
+    std::unique_ptr<Slots> newslots;
+    if (!core_cached_shared_ptr_detail::isDefault(p)) {
+      newslots = std::make_unique<Slots>();
+      // Allocate each Holder in a different CoreRawAllocator stripe to
+      // prevent false sharing. Their control blocks will be adjacent
+      // thanks to allocate_shared().
+      for (size_t i = 0; i < SlotsConfig::num(); ++i) {
+        auto alloc = getCoreAllocator<Holder, kMaxSlots>(i);
+        auto holder = std::allocate_shared<Holder>(alloc, p);
+        newslots->slots[i] = std::shared_ptr<T>(holder, p.get());
+      }
     }
 
-    auto oldslots = slots_.exchange(newslots.release());
-    if (oldslots) {
+    if (auto oldslots = slots_.exchange(newslots.release())) {
       oldslots->retire();
     }
   }
 
   std::shared_ptr<T> get() const {
     folly::hazptr_local<1> hazptr;
-    auto slots = hazptr[0].protect(slots_);
-    if (!slots) {
+    if (auto slots = hazptr[0].protect(slots_)) {
+      return slots->slots[AccessSpreader<>::cachedCurrent(SlotsConfig::num())];
+    } else {
       return nullptr;
     }
-    return (slots->slots_)[AccessSpreader<>::cachedCurrent(kNumSlots)];
   }
 
  private:
   using Holder = std::shared_ptr<T>;
   struct Slots : folly::hazptr_obj_base<Slots> {
-    std::array<std::shared_ptr<T>, kNumSlots> slots_;
+    std::array<std::shared_ptr<T>, kMaxSlots> slots;
   };
   std::atomic<Slots*> slots_{nullptr};
 };

folly/concurrency/test/CoreCachedSharedPtrTest.cpp

@@ -32,6 +32,22 @@
 #include <folly/experimental/ReadMostlySharedPtr.h>
 #include <folly/portability/GTest.h>
 
+namespace {
+
+template <class Operation>
+void parallelRun(
+    Operation op, size_t numThreads = std::thread::hardware_concurrency()) {
+  std::vector<std::thread> threads;
+  for (size_t t = 0; t < numThreads; ++t) {
+    threads.emplace_back([&, t] { op(t); });
+  }
+  for (auto& t : threads) {
+    t.join();
+  }
+}
+
+} // namespace
+
 TEST(CoreCachedSharedPtr, Basic) {
   auto p = std::make_shared<int>(1);
   std::weak_ptr<int> wp(p);
@@ -45,6 +61,13 @@ TEST(CoreCachedSharedPtr, Basic) {
   ASSERT_EQ(*p2, 1);
   ASSERT_FALSE(wp2.expired());
 
+  // Check that other cores get the correct shared_ptr too.
+  parallelRun([&](size_t) {
+    EXPECT_TRUE(cached.get().get() == p.get());
+    EXPECT_EQ(*cached.get(), 1);
+    EXPECT_EQ(*wcached.lock(), 1);
+  });
+
   p.reset();
   cached.reset();
   // p2 should survive.
@@ -57,11 +80,99 @@ TEST(CoreCachedSharedPtr, Basic) {
   ASSERT_TRUE(wp2.expired());
 }
 
+TEST(CoreCachedSharedPtr, AtomicCoreCachedSharedPtr) {
+  constexpr size_t kIters = 2000;
+  {
+    folly::AtomicCoreCachedSharedPtr<size_t> p;
+    parallelRun([&](size_t) {
+      for (size_t i = 0; i < kIters; ++i) {
+        p.reset(std::make_shared<size_t>(i));
+        EXPECT_TRUE(p.get());
+        // Just read the value, and ensure that ASAN/TSAN do not complain.
+        EXPECT_GE(*p.get(), 0);
+      }
+    });
+  }
+  {
+    // One writer thread, all other readers, verify consistency.
+    std::atomic<size_t> largestValueObserved{0};
+    folly::AtomicCoreCachedSharedPtr<size_t> p{std::make_shared<size_t>(0)};
+    parallelRun([&](size_t t) {
+      if (t == 0) {
+        for (size_t i = 0; i < kIters; ++i) {
+          p.reset(std::make_shared<size_t>(i + 1));
+        }
+      } else {
+        while (true) {
+          auto exp = largestValueObserved.load();
+          auto value = *p.get();
+          EXPECT_GE(value, exp);
+          // Maintain the maximum value observed so far. As soon as one thread
+          // observes an update, they all should observe it.
+          while (value > exp &&
+                 !largestValueObserved.compare_exchange_weak(exp, value)) {
+          }
+          if (exp == kIters) {
+            break;
+          }
+        }
+      }
+    });
+  }
+}
+
+namespace {
+
+template <class Holder>
+void testAliasingCornerCases() {
+  {
+    Holder h;
+    auto p1 = std::make_shared<int>(1);
+    std::weak_ptr<int> w1 = p1;
+    // Aliasing constructor, p2 is nullptr but still manages the object in p1.
+    std::shared_ptr<int> p2(p1, nullptr);
+    // And now it's the only reference left.
+    p1.reset();
+    EXPECT_FALSE(w1.expired());
+
+    // Pass the ownership to the Holder.
+    h.reset(p2);
+    p2.reset();
+
+    // Object should still be alive.
+    EXPECT_FALSE(w1.expired());
+
+    // And resetting will destroy it.
+    h.reset();
+    folly::hazptr_cleanup();
+    EXPECT_TRUE(w1.expired());
+  }
+
+  {
+    Holder h;
+    int x = 1;
+    // p points to x, but has no managed object.
+    std::shared_ptr<int> p(std::shared_ptr<int>{}, &x);
+    h.reset(p);
+    EXPECT_TRUE(h.get().get() == &x);
+  }
+}
+
+} // namespace
+
+TEST(CoreCachedSharedPtr, AliasingCornerCases) {
+  testAliasingCornerCases<folly::CoreCachedSharedPtr<int>>();
+}
+
+TEST(CoreCachedSharedPtr, AliasingCornerCasesAtomic) {
+  testAliasingCornerCases<folly::AtomicCoreCachedSharedPtr<int>>();
+}
+
 namespace {
 
 template <class Operation>
-size_t parallelRun(Operation op, size_t numThreads) {
-  constexpr size_t kIters = 1000 * 1000;
+size_t benchmarkParallelRun(Operation op, size_t numThreads) {
+  constexpr size_t kIters = 2 * 1000 * 1000;
 
   std::vector<std::thread> threads;
 
@@ -97,47 +208,47 @@ size_t parallelRun(Operation op, size_t numThreads) {
 
 size_t benchmarkSharedPtrAcquire(size_t numThreads) {
   auto p = std::make_shared<int>(1);
-  return parallelRun([&] { return p; }, numThreads);
+  return benchmarkParallelRun([&] { return p; }, numThreads);
 }
 
 size_t benchmarkWeakPtrLock(size_t numThreads) {
   auto p = std::make_shared<int>(1);
   std::weak_ptr<int> wp = p;
-  return parallelRun([&] { return wp.lock(); }, numThreads);
+  return benchmarkParallelRun([&] { return wp.lock(); }, numThreads);
 }
 
 size_t benchmarkAtomicSharedPtrAcquire(size_t numThreads) {
   auto s = std::make_shared<int>(1);
   folly::atomic_shared_ptr<int> p;
   p.store(s);
-  return parallelRun([&] { return p.load(); }, numThreads);
+  return benchmarkParallelRun([&] { return p.load(); }, numThreads);
 }
 
 size_t benchmarkCoreCachedSharedPtrAcquire(size_t numThreads) {
   folly::CoreCachedSharedPtr<int> p(std::make_shared<int>(1));
-  return parallelRun([&] { return p.get(); }, numThreads);
+  return benchmarkParallelRun([&] { return p.get(); }, numThreads);
 }
 
 size_t benchmarkCoreCachedWeakPtrLock(size_t numThreads) {
   folly::CoreCachedSharedPtr<int> p(std::make_shared<int>(1));
   folly::CoreCachedWeakPtr<int> wp(p);
-  return parallelRun([&] { return wp.lock(); }, numThreads);
+  return benchmarkParallelRun([&] { return wp.lock(); }, numThreads);
 }
 
 size_t benchmarkAtomicCoreCachedSharedPtrAcquire(size_t numThreads) {
   folly::AtomicCoreCachedSharedPtr<int> p(std::make_shared<int>(1));
-  return parallelRun([&] { return p.get(); }, numThreads);
+  return benchmarkParallelRun([&] { return p.get(); }, numThreads);
 }
 
 size_t benchmarkReadMostlySharedPtrAcquire(size_t numThreads) {
   folly::ReadMostlyMainPtr<int> p{std::make_shared<int>(1)};
-  return parallelRun([&] { return p.getShared(); }, numThreads);
+  return benchmarkParallelRun([&] { return p.getShared(); }, numThreads);
 }
 
 size_t benchmarkReadMostlyWeakPtrLock(size_t numThreads) {
   folly::ReadMostlyMainPtr<int> p{std::make_shared<int>(1)};
   folly::ReadMostlyWeakPtr<int> w{p};
-  return parallelRun([&] { return w.lock(); }, numThreads);
+  return benchmarkParallelRun([&] { return w.lock(); }, numThreads);
 }
 
 } // namespace
@@ -178,21 +289,21 @@ BENCHMARK_THREADS(64)
 
 BENCHMARK_MULTI(SharedPtrSingleThreadReset) {
   auto p = std::make_shared<int>(1);
-  return parallelRun([&] { p = std::make_shared<int>(1); }, 1);
+  return benchmarkParallelRun([&] { p = std::make_shared<int>(1); }, 1);
 }
 BENCHMARK_MULTI(AtomicSharedPtrSingleThreadReset) {
   auto s = std::make_shared<int>(1);
   folly::atomic_shared_ptr<int> p;
   p.store(s);
-  return parallelRun([&] { p.store(std::make_shared<int>(1)); }, 1);
+  return benchmarkParallelRun([&] { p.store(std::make_shared<int>(1)); }, 1);
 }
 BENCHMARK_MULTI(CoreCachedSharedPtrSingleThreadReset) {
   folly::CoreCachedSharedPtr<int> p(std::make_shared<int>(1));
-  return parallelRun([&] { p.reset(std::make_shared<int>(1)); }, 1);
+  return benchmarkParallelRun([&] { p.reset(std::make_shared<int>(1)); }, 1);
 }
 BENCHMARK_MULTI(AtomicCoreCachedSharedPtrSingleThreadReset) {
   folly::AtomicCoreCachedSharedPtr<int> p(std::make_shared<int>(1));
-  return parallelRun([&] { p.reset(std::make_shared<int>(1)); }, 1);
+  return benchmarkParallelRun([&] { p.reset(std::make_shared<int>(1)); }, 1);
 }
 
 int main(int argc, char** argv) {
@@ -216,63 +327,63 @@ $ buck-out/gen/folly/concurrency/test/core_cached_shared_ptr_test --benchmark --
 ============================================================================
 folly/concurrency/test/CoreCachedSharedPtrTest.cpprelative  time/iter  iters/s
 ============================================================================
-SharedPtrAcquire_1Threads                                   19.89ns   50.28M
-WeakPtrLock_1Threads                                        22.21ns   45.02M
-AtomicSharedPtrAcquire_1Threads                             27.50ns   36.37M
-CoreCachedSharedPtrAquire_1Threads                          19.36ns   51.65M
-CoreCachedWeakPtrLock_1Threads                              22.07ns   45.31M
-AtomicCoreCachedSharedPtrAcquire_1Threads                   22.68ns   44.09M
-ReadMostlySharedPtrAcquire_1Threads                         20.27ns   49.34M
-ReadMostlyWeakPtrLock_1Threads                              20.23ns   49.43M
+SharedPtrAcquire_1Threads                                   18.31ns   54.62M
+WeakPtrLock_1Threads                                        20.07ns   49.83M
+AtomicSharedPtrAcquire_1Threads                             26.31ns   38.01M
+CoreCachedSharedPtrAquire_1Threads                          20.17ns   49.57M
+CoreCachedWeakPtrLock_1Threads                              22.41ns   44.62M
+AtomicCoreCachedSharedPtrAcquire_1Threads                   23.59ns   42.40M
+ReadMostlySharedPtrAcquire_1Threads                         20.16ns   49.61M
+ReadMostlyWeakPtrLock_1Threads                              20.46ns   48.87M
 ----------------------------------------------------------------------------
-SharedPtrAcquire_4Threads                                  187.84ns    5.32M
-WeakPtrLock_4Threads                                       207.78ns    4.81M
-AtomicSharedPtrAcquire_4Threads                            552.59ns    1.81M
-CoreCachedSharedPtrAquire_4Threads                          20.77ns   48.14M
-CoreCachedWeakPtrLock_4Threads                              23.17ns   43.15M
-AtomicCoreCachedSharedPtrAcquire_4Threads                   23.85ns   41.94M
-ReadMostlySharedPtrAcquire_4Threads                         22.20ns   45.05M
-ReadMostlyWeakPtrLock_4Threads                              21.79ns   45.89M
+SharedPtrAcquire_4Threads                                  193.62ns    5.16M
+WeakPtrLock_4Threads                                       216.29ns    4.62M
+AtomicSharedPtrAcquire_4Threads                            508.04ns    1.97M
+CoreCachedSharedPtrAquire_4Threads                          21.52ns   46.46M
+CoreCachedWeakPtrLock_4Threads                              23.80ns   42.01M
+AtomicCoreCachedSharedPtrAcquire_4Threads                   24.81ns   40.30M
+ReadMostlySharedPtrAcquire_4Threads                         21.67ns   46.15M
+ReadMostlyWeakPtrLock_4Threads                              21.72ns   46.04M
 ----------------------------------------------------------------------------
-SharedPtrAcquire_8Threads                                  341.50ns    2.93M
-WeakPtrLock_8Threads                                       463.56ns    2.16M
-AtomicSharedPtrAcquire_8Threads                              1.29us  776.90K
-CoreCachedSharedPtrAquire_8Threads                          20.92ns   47.81M
-CoreCachedWeakPtrLock_8Threads                              23.31ns   42.91M
-AtomicCoreCachedSharedPtrAcquire_8Threads                   23.84ns   41.94M
-ReadMostlySharedPtrAcquire_8Threads                         21.57ns   46.36M
-ReadMostlyWeakPtrLock_8Threads                              21.62ns   46.26M
+SharedPtrAcquire_8Threads                                  389.71ns    2.57M
+WeakPtrLock_8Threads                                       467.29ns    2.14M
+AtomicSharedPtrAcquire_8Threads                              1.38us  727.03K
+CoreCachedSharedPtrAquire_8Threads                          21.49ns   46.53M
+CoreCachedWeakPtrLock_8Threads                              23.83ns   41.97M
+AtomicCoreCachedSharedPtrAcquire_8Threads                   24.68ns   40.52M
+ReadMostlySharedPtrAcquire_8Threads                         21.68ns   46.12M
+ReadMostlyWeakPtrLock_8Threads                              21.48ns   46.55M
 ----------------------------------------------------------------------------
-SharedPtrAcquire_16Threads                                 855.57ns    1.17M
-WeakPtrLock_16Threads                                      898.42ns    1.11M
-AtomicSharedPtrAcquire_16Threads                             3.28us  304.83K
-CoreCachedSharedPtrAquire_16Threads                         22.55ns   44.35M
-CoreCachedWeakPtrLock_16Threads                             24.73ns   40.44M
-AtomicCoreCachedSharedPtrAcquire_16Threads                  26.24ns   38.10M
-ReadMostlySharedPtrAcquire_16Threads                        24.44ns   40.92M
-ReadMostlyWeakPtrLock_16Threads                             24.60ns   40.65M
+SharedPtrAcquire_16Threads                                 739.59ns    1.35M
+WeakPtrLock_16Threads                                      896.23ns    1.12M
+AtomicSharedPtrAcquire_16Threads                             2.88us  347.73K
+CoreCachedSharedPtrAquire_16Threads                         21.98ns   45.50M
+CoreCachedWeakPtrLock_16Threads                             25.98ns   38.49M
+AtomicCoreCachedSharedPtrAcquire_16Threads                  26.44ns   37.82M
+ReadMostlySharedPtrAcquire_16Threads                        23.75ns   42.11M
+ReadMostlyWeakPtrLock_16Threads                             22.89ns   43.70M
 ----------------------------------------------------------------------------
-SharedPtrAcquire_32Threads                                   1.39us  717.39K
-WeakPtrLock_32Threads                                        2.02us  494.63K
-AtomicSharedPtrAcquire_32Threads                             4.97us  201.20K
-CoreCachedSharedPtrAquire_32Threads                         30.78ns   32.49M
-CoreCachedWeakPtrLock_32Threads                             27.89ns   35.85M
-AtomicCoreCachedSharedPtrAcquire_32Threads                  30.56ns   32.72M
+SharedPtrAcquire_32Threads                                   1.36us  732.78K
+WeakPtrLock_32Threads                                        1.93us  518.58K
+AtomicSharedPtrAcquire_32Threads                             5.68us  176.04K
+CoreCachedSharedPtrAquire_32Threads                         29.24ns   34.20M
+CoreCachedWeakPtrLock_32Threads                             32.17ns   31.08M
+AtomicCoreCachedSharedPtrAcquire_32Threads                  28.67ns   34.88M
 ReadMostlySharedPtrAcquire_32Threads                        29.36ns   34.06M
-ReadMostlyWeakPtrLock_32Threads                             29.63ns   33.75M
+ReadMostlyWeakPtrLock_32Threads                             27.27ns   36.67M
 ----------------------------------------------------------------------------
-SharedPtrAcquire_64Threads                                   2.49us  402.30K
-WeakPtrLock_64Threads                                        4.57us  218.74K
-AtomicSharedPtrAcquire_64Threads                             9.78us  102.28K
-CoreCachedSharedPtrAquire_64Threads                         48.75ns   20.51M
-CoreCachedWeakPtrLock_64Threads                             52.85ns   18.92M
-AtomicCoreCachedSharedPtrAcquire_64Threads                  56.58ns   17.67M
-ReadMostlySharedPtrAcquire_64Threads                        56.58ns   17.68M
-ReadMostlyWeakPtrLock_64Threads                             56.87ns   17.59M
+SharedPtrAcquire_64Threads                                   2.39us  418.35K
+WeakPtrLock_64Threads                                        4.21us  237.61K
+AtomicSharedPtrAcquire_64Threads                             8.63us  115.86K
+CoreCachedSharedPtrAquire_64Threads                         49.70ns   20.12M
+CoreCachedWeakPtrLock_64Threads                             74.74ns   13.38M
+AtomicCoreCachedSharedPtrAcquire_64Threads                  56.09ns   17.83M
+ReadMostlySharedPtrAcquire_64Threads                        49.22ns   20.32M
+ReadMostlyWeakPtrLock_64Threads                             49.16ns   20.34M
 ----------------------------------------------------------------------------
-SharedPtrSingleThreadReset                                  10.50ns   95.28M
-AtomicSharedPtrSingleThreadReset                            44.02ns   22.72M
-CoreCachedSharedPtrSingleThreadReset                         4.57us  218.87K
-AtomicCoreCachedSharedPtrSingleThreadReset                   5.22us  191.47K
+SharedPtrSingleThreadReset                                  10.45ns   95.70M
+AtomicSharedPtrSingleThreadReset                            42.83ns   23.35M
+CoreCachedSharedPtrSingleThreadReset                         2.51us  398.43K
+AtomicCoreCachedSharedPtrSingleThreadReset                   2.36us  423.31K
 ============================================================================
 #endif