ConcurrentHashMap: Optimize iteration, empty, size, destruction.

Summary:
## Optimizations:
- Avoid unnecessary creation of empty segments when iterating.
- Keep track of the lowest and highest nonempty segments.

## Improvements:
- Construction/destruction:
```
CHM ctor/dtor -- empty              213 ns    200 ns    196 ns
CHM ctor/dtor -- 1 item             395 ns    388 ns    373 ns
```
```
CHM ctor/dtor -- empty               99 ns     96 ns     94 ns
CHM ctor/dtor -- 1 item             218 ns    217 ns    216 ns
```
- begin() before and after:
```
CHM begin() -- empty                705 ns    689 ns    660 ns
```
```
CHM begin() -- empty                  6 ns      6 ns      6 ns
```
- Iteration before and after:
```
CHM iterate -- empty                722 ns    714 ns    707 ns
CHM iterate -- 1 item               621 ns    601 ns    587 ns
CHM iterate -- 10 items              89 ns     88 ns     84 ns
CHM iterate -- 100 items             55 ns     51 ns     48 ns
CHM iterate -- 1K items              15 ns     14 ns     13 ns
CHM iterate -- 10K items             13 ns     13 ns     12 ns
CHM iterate -- 100K items            13 ns     12 ns     11 ns
CHM iterate -- 1M items              36 ns     29 ns     27 ns
CHM iterate -- 10M items             36 ns     35 ns     34 ns
```
```
CHM iterate -- empty                  7 ns      6 ns      6 ns
CHM iterate -- 1 item                33 ns     32 ns     31 ns
CHM iterate -- 10 items              27 ns     26 ns     26 ns
CHM iterate -- 100 items             34 ns     33 ns     33 ns
CHM iterate -- 1K items              10 ns     10 ns     10 ns
CHM iterate -- 10K items             10 ns      9 ns      8 ns
CHM iterate -- 100K items             9 ns      7 ns      7 ns
CHM iterate -- 1M items              24 ns     19 ns     17 ns
CHM iterate -- 10M items             27 ns     24 ns     23 ns
```
- empty() before and after:
```
CHM empty() -- empty                106 ns    104 ns    102 ns
```
```
CHM empty() -- empty                  1 ns      1 ns      1 ns
```
- size() before and after:
```
CHM size() -- empty                 104 ns    101 ns     99 ns
CHM size() -- 1 item                103 ns    101 ns     96 ns
CHM size() -- 10 items              107 ns    106 ns    104 ns
CHM size() -- 100 items             111 ns    100 ns     95 ns
```
```
CHM size() -- empty                   1 ns      1 ns      1 ns
CHM size() -- 1 item                  2 ns      2 ns      2 ns
CHM size() -- 10 items                8 ns      8 ns      8 ns
CHM size() -- 100 items              88 ns     71 ns     68 ns
```

Reviewed By: Gownta

Differential Revision: D32684489

fbshipit-source-id: fb58277d6e58de0d95cca3c4274e1b08d182a45b

folly/concurrency/ConcurrentHashMap.h

@@ -222,6 +222,12 @@ class ConcurrentHashMap {
     }
     cohort_.store(o.cohort(), std::memory_order_relaxed);
     o.cohort_.store(nullptr, std::memory_order_relaxed);
+    beginSeg_.store(
+        o.beginSeg_.load(std::memory_order_relaxed), std::memory_order_relaxed);
+    o.beginSeg_.store(NumShards, std::memory_order_relaxed);
+    endSeg_.store(
+        o.endSeg_.load(std::memory_order_relaxed), std::memory_order_relaxed);
+    o.endSeg_.store(0, std::memory_order_relaxed);
   }
 
   ConcurrentHashMap& operator=(ConcurrentHashMap&& o) {
@@ -241,11 +247,19 @@ class ConcurrentHashMap {
     cohort_shutdown_cleanup();
     cohort_.store(o.cohort(), std::memory_order_relaxed);
     o.cohort_.store(nullptr, std::memory_order_relaxed);
+    beginSeg_.store(
+        o.beginSeg_.load(std::memory_order_relaxed), std::memory_order_relaxed);
+    o.beginSeg_.store(NumShards, std::memory_order_relaxed);
+    endSeg_.store(
+        o.endSeg_.load(std::memory_order_relaxed), std::memory_order_relaxed);
+    o.endSeg_.store(0, std::memory_order_relaxed);
     return *this;
   }
 
   ~ConcurrentHashMap() {
-    for (uint64_t i = 0; i < NumShards; i++) {
+    uint64_t begin = beginSeg_.load(std::memory_order_acquire);
+    uint64_t end = endSeg_.load(std::memory_order_acquire);
+    for (uint64_t i = begin; i < end; ++i) {
       auto seg = segments_[i].load(std::memory_order_relaxed);
       if (seg) {
         seg->~SegmentT();
@@ -256,7 +270,12 @@ class ConcurrentHashMap {
   }
 
   bool empty() const noexcept {
-    for (uint64_t i = 0; i < NumShards; i++) {
+    uint64_t begin = beginSeg_.load(std::memory_order_acquire);
+    uint64_t end = endSeg_.load(std::memory_order_acquire);
+    // Note: beginSeg_ and endSeg_ are only conservative hints of the
+    // range of non-empty segments. This function cannot conclude that
+    // a map is nonempty merely because beginSeg_ < endSeg_.
+    for (uint64_t i = begin; i < end; ++i) {
       auto seg = segments_[i].load(std::memory_order_acquire);
       if (seg) {
         if (!seg->empty()) {
@@ -425,7 +444,7 @@ class ConcurrentHashMap {
     auto segment = pickSegment(pos->first);
     ConstIterator res(this, segment);
     ensureSegment(segment)->erase(res.it_, pos.it_);
-    res.next(); // May point to segment end, and need to advance.
+    res.advanceIfAtSegmentEnd();
     return res;
   }
 
@@ -457,7 +476,9 @@ class ConcurrentHashMap {
 
   // NOT noexcept, initializes new shard segments vs.
   void clear() {
-    for (uint64_t i = 0; i < NumShards; i++) {
+    uint64_t begin = beginSeg_.load(std::memory_order_acquire);
+    uint64_t end = endSeg_.load(std::memory_order_acquire);
+    for (uint64_t i = begin; i < end; ++i) {
       auto seg = segments_[i].load(std::memory_order_acquire);
       if (seg) {
         seg->clear();
@@ -469,7 +490,9 @@ class ConcurrentHashMap {
     count = count >> ShardBits;
     if (!count)
       return;
-    for (uint64_t i = 0; i < NumShards; i++) {
+    uint64_t begin = beginSeg_.load(std::memory_order_acquire);
+    uint64_t end = endSeg_.load(std::memory_order_acquire);
+    for (uint64_t i = begin; i < end; ++i) {
       auto seg = segments_[i].load(std::memory_order_acquire);
       if (seg) {
         seg->rehash(count);
@@ -480,7 +503,9 @@ class ConcurrentHashMap {
   // This is a rolling size, and is not exact at any moment in time.
   size_t size() const noexcept {
     size_t res = 0;
-    for (uint64_t i = 0; i < NumShards; i++) {
+    uint64_t begin = beginSeg_.load(std::memory_order_acquire);
+    uint64_t end = endSeg_.load(std::memory_order_acquire);
+    for (uint64_t i = begin; i < end; ++i) {
       auto seg = segments_[i].load(std::memory_order_acquire);
       if (seg) {
         res += seg->size();
@@ -492,7 +517,9 @@ class ConcurrentHashMap {
   float max_load_factor() const { return load_factor_; }
 
   void max_load_factor(float factor) {
-    for (uint64_t i = 0; i < NumShards; i++) {
+    uint64_t begin = beginSeg_.load(std::memory_order_acquire);
+    uint64_t end = endSeg_.load(std::memory_order_acquire);
+    for (uint64_t i = begin; i < end; ++i) {
       auto seg = segments_[i].load(std::memory_order_acquire);
       if (seg) {
         seg->max_load_factor(factor);
@@ -510,7 +537,7 @@ class ConcurrentHashMap {
 
     ConstIterator& operator++() {
       ++it_;
-      next();
+      advanceIfAtSegmentEnd();
       return *this;
     }
 
@@ -544,32 +571,43 @@ class ConcurrentHashMap {
    private:
     // cbegin iterator
     explicit ConstIterator(const ConcurrentHashMap* parent)
-        : it_(parent->ensureSegment(0)->cbegin()),
-          segment_(0),
+        : it_(nullptr),
+          segment_(parent->beginSeg_.load(std::memory_order_acquire)),
           parent_(parent) {
-      // Always iterate to the first element, could be in any shard.
-      next();
+      advanceToSegmentBegin();
     }
 
     // cend iterator
     explicit ConstIterator(uint64_t shards) : it_(nullptr), segment_(shards) {}
 
-    void next() {
-      while (segment_ < parent_->NumShards &&
-             it_ == parent_->ensureSegment(segment_)->cend()) {
-        SegmentT* seg{nullptr};
-        while (!seg) {
-          segment_++;
-          if (segment_ < parent_->NumShards) {
-            seg = parent_->segments_[segment_].load(std::memory_order_acquire);
-            if (!seg) {
-              continue;
-            }
-            it_ = seg->cbegin();
+    void advanceIfAtSegmentEnd() {
+      DCHECK_LT(segment_, parent_->NumShards);
+      SegmentT* seg =
+          parent_->segments_[segment_].load(std::memory_order_acquire);
+      DCHECK(seg);
+      if (it_ == seg->cend()) {
+        ++segment_;
+        advanceToSegmentBegin();
+      }
+    }
+
+    FOLLY_ALWAYS_INLINE void advanceToSegmentBegin() {
+      // Advance to the beginning of the next nonempty segment
+      // starting from segment_.
+      uint64_t end = parent_->endSeg_.load(std::memory_order_acquire);
+      while (segment_ < end) {
+        SegmentT* seg =
+            parent_->segments_[segment_].load(std::memory_order_acquire);
+        if (seg) {
+          it_ = seg->cbegin();
+          if (it_ != seg->cend()) {
+            return;
           }
-          break;
         }
+        ++segment_;
       }
+      // All segments are empty. Advance to end.
+      segment_ = parent_->NumShards;
     }
 
     typename SegmentT::Iterator it_;
@@ -658,11 +696,26 @@ class ConcurrentHashMap {
         Allocator().deallocate((uint8_t*)newseg, sizeof(SegmentT));
       } else {
         seg = newseg;
+        updateBeginAndEndSegments(i);
       }
     }
     return seg;
   }
 
+  void updateBeginAndEndSegments(uint64_t i) const {
+    uint64_t val = beginSeg_.load(std::memory_order_acquire);
+    while (i < val && !casSeg(beginSeg_, val, i)) {
+    }
+    val = endSeg_.load(std::memory_order_acquire);
+    while (i + 1 > val && !casSeg(endSeg_, val, i + 1)) {
+    }
+  }
+
+  bool casSeg(Atom<uint64_t>& seg, uint64_t& expval, uint64_t newval) const {
+    return seg.compare_exchange_weak(
+        expval, newval, std::memory_order_acq_rel, std::memory_order_acquire);
+  }
+
   hazptr_obj_cohort<Atom>* cohort() const noexcept {
     return cohort_.load(std::memory_order_acquire);
   }
@@ -694,6 +747,8 @@ class ConcurrentHashMap {
   size_t size_{0};
   size_t max_size_{0};
   mutable Atom<hazptr_obj_cohort<Atom>*> cohort_{nullptr};
+  mutable Atom<uint64_t> beginSeg_{NumShards};
+  mutable Atom<uint64_t> endSeg_{0};
 };
 
 #if FOLLY_SSE_PREREQ(4, 2) && !FOLLY_MOBILE

folly/concurrency/test/ConcurrentHashMapTest.cpp

@@ -112,6 +112,10 @@ TYPED_TEST_P(ConcurrentHashMapTest, MapTest) {
   EXPECT_FALSE(foomap.empty());
   foomap.clear();
   EXPECT_TRUE(foomap.empty());
+  EXPECT_TRUE(foomap.insert(3, 0).second);
+  EXPECT_FALSE(foomap.empty());
+  foomap.erase(3);
+  EXPECT_TRUE(foomap.empty());
 }
 
 TYPED_TEST_P(ConcurrentHashMapTest, MaxSizeTest) {