experimental: Add single writer multi-reader fixed hash map

Summary:
Add a fixed single-writer multi-reader hash map that supports:
- Copy construction with optional expansion
- Concurrent read-only lookup.
- Concurrent read-only iteration.

The map has fixed size. Higher-level users can manage instances of this map to build a more general unbounded map.

Reviewed By: davidtgoldblatt

Differential Revision: D17522603

fbshipit-source-id: b4fcfe427a343f7226d216670536f2594f187bf3

folly/experimental/SingleWriterFixedHashMap.h

+/*
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <folly/lang/Bits.h>
+#include <glog/logging.h>
+
+#include <atomic>
+
+namespace folly {
+
+/// SingleWriterFixedHashMap:
+///
+/// Minimal single-writer fixed hash map implementation that supports:
+/// - Copy construction with optional capacity expansion.
+/// - Concurrent read-only lookup.
+/// - Concurrent read-only iteration.
+///
+/// Assumes that higher level code:
+/// - Checks availability of empty slots before calling insert
+/// - Manages expansion and/or cleanup of tombstones
+///
+/// Notes on algorithm:
+/// - Tombstones are used to mark previously occupied slots.
+/// - A slot with a tombstone can only be reused for the same key. The
+///   reason for that is to enforce that once a key occupies a slot,
+///   that key cannot use any other slot for the lifetime of the
+///   map. This is to guarantee that when readers iterate over the map
+///   they do not encounter any key more than once.
+///
+/// Writer-only operations:
+/// - insert()
+/// - erase()
+/// - used()
+/// - available()
+///
+template <typename Key, typename Value>
+class SingleWriterFixedHashMap {
+#if !FOLLY_MOBILE
+  static_assert(
+      std::atomic<Value>::is_always_lock_free,
+      "This implementation depends on having fast atomic "
+      "data-race-free loads and stores of Value type.");
+#endif
+  static_assert(
+      std::is_trivial<Key>::value,
+      "This implementation depends on using a single key instance "
+      "for all insert and erase operations. The reason is to allow "
+      "readers to read keys data-race-free concurrently with possible "
+      "concurrent insert and erase operations on the keys.");
+
+  class Elem;
+
+  enum class State : uint8_t { EMPTY, VALID, TOMBSTONE };
+
+  size_t capacity_;
+  size_t used_{0};
+  std::atomic<size_t> size_{0};
+  std::unique_ptr<Elem[]> elem_;
+
+ public:
+  class Iterator;
+
+  explicit SingleWriterFixedHashMap(size_t capacity)
+      : capacity_(folly::nextPowTwo(capacity)) {}
+
+  explicit SingleWriterFixedHashMap(
+      size_t capacity,
+      const SingleWriterFixedHashMap& o)
+      : capacity_(folly::nextPowTwo(capacity)) {
+    if (o.empty()) {
+      return;
+    }
+    elem_ = std::make_unique<Elem[]>(capacity_);
+    for (size_t i = 0; i < o.capacity_; ++i) {
+      Elem& e = o.elem_[i];
+      if (e.valid()) {
+        insert(e.key(), e.value());
+      }
+    }
+  }
+
+  FOLLY_ALWAYS_INLINE Iterator begin() const {
+    return empty() ? end() : Iterator(*this);
+  }
+
+  FOLLY_ALWAYS_INLINE Iterator end() const {
+    return Iterator(*this, capacity_);
+  }
+
+  size_t capacity() const {
+    return capacity_;
+  }
+
+  /* not data-race-free, to be called only by the single writer */
+  size_t used() const {
+    return used_;
+  }
+
+  /* not-data race-free, to be called only by the single writer */
+  size_t available() const {
+    return capacity_ - used_;
+  }
+
+  /* data-race-free, can be called by readers */
+  FOLLY_ALWAYS_INLINE size_t size() const {
+    return size_.load(std::memory_order_acquire);
+  }
+
+  FOLLY_ALWAYS_INLINE bool empty() const {
+    return size() == 0;
+  }
+
+  bool insert(Key key, Value value) {
+    if (!elem_) {
+      elem_ = std::make_unique<Elem[]>(capacity_);
+    }
+    DCHECK_LT(used_, capacity_);
+    if (writer_find(key) < capacity_) {
+      return false;
+    }
+    size_t index = hash(key);
+    auto attempts = capacity_;
+    size_t mask = capacity_ - 1;
+    while (attempts--) {
+      Elem& e = elem_[index];
+      auto state = e.state();
+      if (state == State::EMPTY ||
+          (state == State::TOMBSTONE && e.key() == key)) {
+        if (state == State::EMPTY) {
+          e.setKey(key);
+          ++used_;
+          DCHECK_LE(used_, capacity_);
+        }
+        e.setValue(value);
+        e.setValid();
+        setSize(size() + 1);
+        DCHECK_LE(size(), used_);
+        return true;
+      }
+      index = (index + 1) & mask;
+    }
+    CHECK(false) << "No available slots";
+    folly::assume_unreachable();
+  }
+
+  void erase(Iterator& it) {
+    DCHECK_NE(it, end());
+    Elem& e = elem_[it.index_];
+    erase_internal(e);
+  }
+
+  bool erase(Key key) {
+    size_t index = writer_find(key);
+    if (index == capacity_) {
+      return false;
+    }
+    Elem& e = elem_[index];
+    erase_internal(e);
+    return true;
+  }
+
+  FOLLY_ALWAYS_INLINE Iterator find(Key key) const {
+    size_t index = reader_find(key);
+    return Iterator(*this, index);
+  }
+
+  FOLLY_ALWAYS_INLINE bool contains(Key key) const {
+    return reader_find(key) < capacity_;
+  }
+
+ private:
+  FOLLY_ALWAYS_INLINE size_t hash(Key key) const {
+    size_t mask = capacity_ - 1;
+    size_t index = std::hash<Key>()(key) & mask;
+    DCHECK_LT(index, capacity_);
+    return index;
+  }
+
+  void setSize(size_t size) {
+    size_.store(size, std::memory_order_release);
+  }
+
+  FOLLY_ALWAYS_INLINE size_t reader_find(Key key) const {
+    return find_internal(key);
+  }
+
+  size_t writer_find(Key key) {
+    return find_internal(key);
+  }
+
+  FOLLY_ALWAYS_INLINE size_t find_internal(Key key) const {
+    if (!empty()) {
+      size_t index = hash(key);
+      auto attempts = capacity_;
+      size_t mask = capacity_ - 1;
+      while (attempts--) {
+        Elem& e = elem_[index];
+        auto state = e.state();
+        if (state == State::VALID && e.key() == key) {
+          return index;
+        }
+        if (state == State::EMPTY) {
+          break;
+        }
+        index = (index + 1) & mask;
+      }
+    }
+    return capacity_;
+  }
+
+  void erase_internal(Elem& e) {
+    e.erase();
+    DCHECK_GT(size(), 0);
+    setSize(size() - 1);
+  }
+
+  /// Elem
+  class Elem {
+    std::atomic<State> state_;
+    Key key_;
+    std::atomic<Value> value_;
+
+   public:
+    Elem() : state_(State::EMPTY) {}
+
+    FOLLY_ALWAYS_INLINE State state() const {
+      return state_.load(std::memory_order_acquire);
+    }
+
+    FOLLY_ALWAYS_INLINE bool valid() const {
+      return state() == State::VALID;
+    }
+
+    FOLLY_ALWAYS_INLINE Key key() const {
+      return key_;
+    }
+
+    FOLLY_ALWAYS_INLINE Value value() const {
+      return value_.load(std::memory_order_relaxed);
+    }
+
+    void setKey(Key key) {
+      key_ = key;
+    }
+
+    void setValue(Value value) {
+      value_.store(value, std::memory_order_relaxed);
+    }
+
+    void setValid() {
+      state_.store(State::VALID, std::memory_order_release);
+    }
+
+    void erase() {
+      state_.store(State::TOMBSTONE, std::memory_order_release);
+    }
+  }; // Elem
+
+ public:
+  /// Iterator
+  class Iterator {
+    Elem* elem_;
+    size_t capacity_;
+    size_t index_;
+
+   public:
+    FOLLY_ALWAYS_INLINE Key key() const {
+      DCHECK_LT(index_, capacity_);
+      Elem& e = elem_[index_];
+      return e.key();
+    }
+
+    FOLLY_ALWAYS_INLINE Value value() const {
+      DCHECK_LT(index_, capacity_);
+      Elem& e = elem_[index_];
+      return e.value();
+    }
+
+    FOLLY_ALWAYS_INLINE Iterator& operator++() {
+      DCHECK_LT(index_, capacity_);
+      ++index_;
+      next();
+      return *this;
+    }
+
+    FOLLY_ALWAYS_INLINE bool operator==(const Iterator& o) const {
+      DCHECK(elem_ == o.elem_ || elem_ == nullptr || o.elem_ == nullptr);
+      DCHECK_EQ(capacity_, o.capacity_);
+      DCHECK_LE(index_, capacity_);
+      return index_ == o.index_;
+    }
+
+    FOLLY_ALWAYS_INLINE bool operator!=(const Iterator& o) const {
+      return !(*this == o);
+    }
+
+   private:
+    friend class SingleWriterFixedHashMap;
+
+    explicit Iterator(const SingleWriterFixedHashMap& m, size_t i = 0)
+        : elem_(m.elem_.get()), capacity_(m.capacity_), index_(i) {
+      if (index_ < capacity_) {
+        next();
+      }
+    }
+
+    FOLLY_ALWAYS_INLINE void next() {
+      while (index_ < capacity_ && !elem_[index_].valid()) {
+        ++index_;
+      }
+    }
+  }; // Iterator
+}; // SingleWriterFixedHashMap
+
+} // namespace folly

folly/experimental/test/SingleWriterHashMapTest.cpp

+/*
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <folly/Benchmark.h>
+#include <folly/container/Array.h>
+#include <folly/experimental/SingleWriterFixedHashMap.h>
+#include <folly/portability/GFlags.h>
+#include <folly/portability/GTest.h>
+#include <folly/synchronization/test/Barrier.h>
+
+#include <boost/thread/barrier.hpp>
+#include <glog/logging.h>
+
+#include <atomic>
+#include <thread>
+
+DEFINE_bool(bench, false, "run benchmark");
+DEFINE_int32(reps, 10, "number of reps");
+DEFINE_int32(ops, 1000000, "number of operations per rep");
+
+using SWFHM = folly::SingleWriterFixedHashMap<int, int>;
+
+void basic_test() {
+  SWFHM m(32);
+
+  ASSERT_EQ(m.used(), 0);
+  ASSERT_FALSE(m.erase(0));
+  ASSERT_EQ(m.size(), 0);
+  ASSERT_EQ(m.find(0), m.end());
+  ASSERT_EQ(m.available(), 32);
+
+  ASSERT_TRUE(m.insert(1, 1));
+  ASSERT_EQ(m.size(), 1);
+  ASSERT_EQ(m.used(), 1);
+  ASSERT_NE(m.find(1), m.end());
+  ASSERT_TRUE(m.contains(1));
+  ASSERT_EQ(m.available(), 31);
+
+  ASSERT_TRUE(m.insert(2, 2));
+  ASSERT_EQ(m.size(), 2);
+  ASSERT_EQ(m.used(), 2);
+  ASSERT_NE(m.find(2), m.end());
+  ASSERT_EQ(m.available(), 30);
+
+  ASSERT_TRUE(m.erase(1));
+  ASSERT_TRUE(m.erase(2));
+  ASSERT_EQ(m.size(), 0);
+  ASSERT_EQ(m.used(), 2);
+  ASSERT_EQ(m.available(), 30);
+  ASSERT_EQ(m.find(1), m.end());
+  ASSERT_EQ(m.find(2), m.end());
+}
+
+TEST(SingleWriterFixedHashMap, basic) {
+  basic_test();
+}
+
+void iterator_test() {
+  SWFHM m(32);
+  for (int i = 0; i < 20; ++i) {
+    m.insert(i, i);
+  }
+  for (int i = 0; i < 10; ++i) {
+    m.erase(i);
+  }
+  int sum = 0;
+  for (auto it = m.begin(); it != m.end(); ++it) {
+    sum += it.value();
+  }
+  ASSERT_EQ(sum, 145);
+}
+
+TEST(SingleWriterFixedHashMap, iterator) {
+  iterator_test();
+}
+
+void copy_test() {
+  SWFHM m1(32);
+  for (int i = 0; i < 20; ++i) {
+    m1.insert(i, i);
+  }
+  SWFHM m2(64, m1);
+  for (int i = 0; i < 10; ++i) {
+    m2.erase(i);
+  }
+  int sum = 0;
+  for (auto it = m2.begin(); it != m2.end(); ++it) {
+    sum += it.value();
+  }
+  ASSERT_EQ(sum, 145);
+}
+
+TEST(SingleWriterFixedHashMap, copy) {
+  copy_test();
+}
+
+void drf_test() {
+  SWFHM m(32);
+  int nthr = 5;
+  folly::test::Barrier b1(nthr + 1);
+  std::atomic<bool> stop{false};
+
+  auto writer = std::thread([&] {
+    b1.wait();
+    for (int i = 0; i < 10000; ++i) {
+      for (int j = 0; j < 10; ++j) {
+        m.insert(j, j);
+      }
+      for (int j = 0; j < 10; ++j) {
+        m.erase(j);
+      }
+    }
+    stop.store(true);
+  });
+
+  std::vector<std::thread> readers(nthr - 1);
+  for (int i = 0; i < nthr - 1; ++i) {
+    readers[i] = std::thread([&] {
+      b1.wait();
+      while (!stop) {
+        int sum = 0;
+        for (auto it = m.begin(); it != m.end(); ++it) {
+          sum += it.value();
+        }
+        ASSERT_LE(sum, 45);
+      }
+    });
+  }
+
+  b1.wait();
+  writer.join();
+  for (int i = 0; i < nthr - 1; ++i) {
+    readers[i].join();
+  }
+}
+
+TEST(SingleWriterFixedHashMap, drf) {
+  drf_test();
+}
+
+// Benchmarks
+
+template <typename Func>
+inline uint64_t run_once(int nthr, const Func& fn) {
+  folly::test::Barrier b1(nthr + 1);
+
+  std::vector<std::thread> thr(nthr);
+  for (int tid = 0; tid < nthr; ++tid) {
+    thr[tid] = std::thread([&, tid] {
+      b1.wait();
+      fn(tid);
+    });
+  }
+
+  b1.wait();
+  /* begin time measurement */
+  auto const tbegin = std::chrono::steady_clock::now();
+  /* wait for completion */
+  for (int i = 0; i < nthr; ++i) {
+    thr[i].join();
+  }
+  /* end time measurement */
+  auto const tend = std::chrono::steady_clock::now();
+  auto const dur = tend - tbegin;
+  return std::chrono::duration_cast<std::chrono::nanoseconds>(dur).count();
+}
+
+template <typename RepFunc>
+uint64_t runBench(int ops, const RepFunc& repFn) {
+  uint64_t reps = FLAGS_reps;
+  uint64_t min = UINTMAX_MAX;
+  uint64_t max = 0;
+  uint64_t sum = 0;
+  std::vector<uint64_t> durs(reps);
+  for (uint64_t r = 0; r < reps; ++r) {
+    uint64_t dur = repFn();
+    durs[r] = dur;
+    sum += dur;
+    min = std::min(min, dur);
+    max = std::max(max, dur);
+    // if each rep takes too long run at least 3 reps
+    const uint64_t minute = 60000000000UL;
+    if (sum > minute && r >= 2) {
+      reps = r + 1;
+      break;
+    }
+  }
+  const std::string ns_unit = " ns";
+  uint64_t avg = sum / reps;
+  uint64_t res = min;
+  uint64_t varsum = 0;
+  for (uint64_t r = 0; r < reps; ++r) {
+    auto term = int64_t(reps * durs[r]) - int64_t(sum);
+    varsum += term * term;
+  }
+  uint64_t dev = uint64_t(std::sqrt(varsum) * std::pow(reps, -1.5));
+  std::cout << "   " << std::setw(4) << max / ops << ns_unit;
+  std::cout << "   " << std::setw(4) << avg / ops << ns_unit;
+  std::cout << "   " << std::setw(4) << dev / ops << ns_unit;
+  std::cout << "   " << std::setw(4) << res / ops << ns_unit;
+  std::cout << std::endl;
+  return res;
+}
+
+uint64_t bench_find(const int nthr, const uint64_t ops) {
+  auto repFn = [&] {
+    SWFHM m(64);
+    for (int i = 0; i < 10; ++i) {
+      m.insert(i, i);
+    }
+    auto fn = [&](int tid) {
+      for (uint64_t i = tid; i < 10 * ops; i += nthr) {
+        auto it = m.find(5);
+        if (it.value() != 5) {
+          ASSERT_EQ(it.value(), 5);
+        }
+      }
+    };
+    return run_once(nthr, fn);
+  };
+  return runBench(ops, repFn);
+}
+
+uint64_t bench_iterate(const int nthr, const uint64_t ops) {
+  auto repFn = [&] {
+    SWFHM m(16);
+    for (int i = 0; i < 10; ++i) {
+      m.insert(i, i);
+    }
+    auto fn = [&](int tid) {
+      for (uint64_t i = tid; i < ops; i += nthr) {
+        int sum = 0;
+        for (auto it = m.begin(); it != m.end(); ++it) {
+          sum += it.value();
+        }
+        if (sum != 55) {
+          ASSERT_EQ(sum, 45);
+        }
+      }
+    };
+    return run_once(nthr, fn);
+  };
+  return runBench(ops, repFn);
+}
+
+uint64_t bench_ctor_dtor(const int nthr, const uint64_t ops) {
+  auto repFn = [&] {
+    auto fn = [&](int tid) {
+      for (uint64_t i = tid; i < ops; i += nthr) {
+        SWFHM m(16);
+        folly::doNotOptimizeAway(m);
+      }
+    };
+    return run_once(nthr, fn);
+  };
+  return runBench(ops, repFn);
+}
+
+uint64_t bench_copy_empty_dtor(const int nthr, const uint64_t ops) {
+  auto repFn = [&] {
+    SWFHM m0(16);
+    auto fn = [&](int tid) {
+      for (uint64_t i = tid; i < ops; i += nthr) {
+        SWFHM m(m0.capacity(), m0);
+      }
+    };
+    return run_once(nthr, fn);
+  };
+  return runBench(ops, repFn);
+}
+
+uint64_t bench_copy_nonempty_dtor(const int nthr, const uint64_t ops) {
+  auto repFn = [&] {
+    SWFHM m0(16);
+    m0.insert(10, 10);
+    auto fn = [&](int tid) {
+      for (uint64_t i = tid; i < ops; i += nthr) {
+        SWFHM m(m0.capacity(), m0);
+      }
+    };
+    return run_once(nthr, fn);
+  };
+  return runBench(ops, repFn);
+}
+
+void dottedLine() {
+  std::cout
+      << "........................................................................"
+      << std::endl;
+}
+
+constexpr auto nthr = folly::make_array<int>(1, 10);
+
+TEST(SingleWriterFixedHashMapBench, Bench) {
+  if (!FLAGS_bench) {
+    return;
+  }
+  std::cout
+      << "========================================================================"
+      << std::endl;
+  std::cout << std::setw(2) << FLAGS_reps << " reps of " << std::setw(8)
+            << FLAGS_ops << " operations\n";
+  dottedLine();
+  std::cout << "$ numactl -N 1 $dir/single_writer_hash_map_test --bench\n";
+  std::cout
+      << "========================================================================"
+      << std::endl;
+  std::cout
+      << "Test name                         Max time  Avg time  Dev time  Min time"
+      << std::endl;
+  for (int i : nthr) {
+    std::cout << "============================== " << std::setw(2) << i
+              << " threads "
+              << "==============================" << std::endl;
+    const uint64_t ops = FLAGS_ops;
+    std::cout << "10x find                       ";
+    bench_find(i, ops);
+    std::cout << "iterate 10-element 32-slot map ";
+    bench_iterate(i, ops);
+    std::cout << "construct / destruct           ";
+    bench_ctor_dtor(i, ops);
+    std::cout << "copy empty / destruct          ";
+    bench_copy_empty_dtor(i, ops);
+    std::cout << "copy nonempty / destruct       ";
+    bench_copy_nonempty_dtor(i, ops);
+  }
+  std::cout
+      << "========================================================================"
+      << std::endl;
+}
+
+/*
+========================================================================
+10 reps of  1000000 operations
+........................................................................
+$ numactl -N 1 $dir/single_writer_hash_map_test --bench
+========================================================================
+Test name                         Max time  Avg time  Dev time  Min time
+==============================  1 threads ==============================
+10x find                            36 ns     35 ns      0 ns     34 ns
+iterate 10-element 32-slot map      20 ns     19 ns      0 ns     19 ns
+construct / destruct                 1 ns      1 ns      0 ns      1 ns
+copy empty / destruct                5 ns      4 ns      0 ns      4 ns
+copy nonempty / destruct            39 ns     38 ns      0 ns     37 ns
+============================== 10 threads ==============================
+10x find                             6 ns      4 ns      1 ns      3 ns
+iterate 10-element 32-slot map       3 ns      3 ns      0 ns      1 ns
+construct / destruct                 0 ns      0 ns      0 ns      0 ns
+copy empty / destruct                0 ns      0 ns      0 ns      0 ns
+copy nonempty / destruct             7 ns      5 ns      1 ns      3 ns
+========================================================================
+ */