F14 hash table in folly

Summary:
F14 is a 14-way probing hash table that resolves collisions by double
hashing.  Up to 14 keys are stored in a chunk at a single hash table
position.  SSE2 vector instructions are used to filter within a chunk;
intra-chunk search takes only a handful of instructions.  "F14" refers
to the fact that the algorithm "F"ilters up to "14" keys at a time.
This strategy allows the hash table to be operated at a high maximum
load factor (12/14) while still keeping probe chains very short.

F14 provides compelling replacements for most of the hash tables we use in
production at Facebook.  Switching to it can improve memory efficiency
and performance at the same time.  The hash table implementations
widely deployed in C++ at Facebook exist along a spectrum of space/time
tradeoffs.  The fastest is the least memory efficient, and the most
memory efficient is much slower than the rest.  F14 moves the curve,
simultaneously improving memory efficiency and performance when compared
to the existing algorithms, especially for complex keys and large maps.

Reviewed By: yfeldblum

Differential Revision: D7154343

fbshipit-source-id: 42ebd11b353285855c0fed5dd4b3af4620d39e98

CMakeLists.txt

@@ -407,6 +407,8 @@ if (BUILD_TESTS)
       # EnumerateTest.cpp since it uses macros to define tests.
       #TEST enumerate_test SOURCES EnumerateTest.cpp
       TEST evicting_cache_map_test SOURCES EvictingCacheMapTest.cpp
+      TEST f14_map_test SOURCES F14MapTest.cpp
+      TEST f14_set_test SOURCES F14SetTest.cpp
       TEST foreach_test SOURCES ForeachTest.cpp
       TEST merge_test SOURCES MergeTest.cpp
       TEST sparse_byte_set_test SOURCES SparseByteSetTest.cpp

folly/Makefile.am

@@ -59,9 +59,14 @@ nobase_follyinclude_HEADERS = \
 	container/Access.h \
 	container/Array.h \
 	container/detail/BitIteratorDetail.h \
+	container/detail/F14Memory.h \
+	container/detail/F14Policy.h \
+	container/detail/F14Table.h \
 	container/Iterator.h \
 	container/Enumerate.h \
 	container/EvictingCacheMap.h \
+	container/F14Map.h \
+	container/F14Set.h \
 	container/Foreach.h \
 	container/Foreach-inl.h \
 	container/SparseByteSet.h \
@@ -508,6 +513,7 @@ libfolly_la_SOURCES = \
 	compression/Counters.cpp \
 	compression/Zlib.cpp \
 	concurrency/CacheLocality.cpp \
+	container/detail/F14Table.cpp \
 	detail/AtFork.cpp \
 	detail/Futex.cpp \
 	detail/IPAddress.cpp \

folly/container/detail/F14Memory.h

+/*
+ * Copyright 2017-present Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <cassert>
+#include <cstddef>
+#include <cstdint>
+#include <memory>
+#include <type_traits>
+
+#include <folly/Portability.h>
+
+namespace folly {
+namespace f14 {
+namespace detail {
+
+template <typename Ptr>
+using NonConstPtr = typename std::pointer_traits<Ptr>::template rebind<
+    std::remove_const_t<typename std::pointer_traits<Ptr>::element_type>>;
+
+//////// TaggedPtr
+
+template <typename Ptr>
+class TaggedPtr {
+ public:
+  TaggedPtr() = default;
+  TaggedPtr(TaggedPtr const&) = default;
+  TaggedPtr(TaggedPtr&&) = default;
+  TaggedPtr& operator=(TaggedPtr const&) = default;
+  TaggedPtr& operator=(TaggedPtr&&) = default;
+
+  TaggedPtr(Ptr p, uint8_t e) noexcept : ptr_{p}, extra_{e} {}
+
+  /* implicit */ TaggedPtr(std::nullptr_t) noexcept {}
+
+  TaggedPtr& operator=(std::nullptr_t) noexcept {
+    ptr_ = nullptr;
+    extra_ = 0;
+    return *this;
+  }
+
+  typename std::pointer_traits<Ptr>::element_type& operator*() const noexcept {
+    return *ptr_;
+  }
+
+  typename std::pointer_traits<Ptr>::element_type* operator->() const noexcept {
+    return std::addressof(*ptr_);
+  }
+
+  Ptr ptr() const {
+    return ptr_;
+  }
+
+  void setPtr(Ptr p) {
+    ptr_ = p;
+  }
+
+  uint8_t extra() const {
+    return extra_;
+  }
+
+  void setExtra(uint8_t e) {
+    extra_ = e;
+  }
+
+  bool operator==(TaggedPtr const& rhs) const noexcept {
+    return ptr_ == rhs.ptr_ && extra_ == rhs.extra_;
+  }
+  bool operator!=(TaggedPtr const& rhs) const noexcept {
+    return !(*this == rhs);
+  }
+
+  bool operator<(TaggedPtr const& rhs) const noexcept {
+    return ptr_ != rhs.ptr_ ? ptr_ < rhs.ptr_ : extra_ < rhs.extra_;
+  }
+
+  bool operator==(std::nullptr_t) const noexcept {
+    return ptr_ == nullptr;
+  }
+  bool operator!=(std::nullptr_t) const noexcept {
+    return !(*this == nullptr);
+  }
+
+ private:
+  Ptr ptr_{};
+  uint8_t extra_{};
+};
+
+#if FOLLY_X64 || FOLLY_AARCH64
+
+template <typename T>
+class TaggedPtr<T*> {
+ public:
+  TaggedPtr() = default;
+  TaggedPtr(TaggedPtr const&) = default;
+  TaggedPtr(TaggedPtr&&) = default;
+  TaggedPtr& operator=(TaggedPtr const&) = default;
+  TaggedPtr& operator=(TaggedPtr&&) = default;
+
+  TaggedPtr(T* p, uint8_t e) noexcept
+      : raw_{(reinterpret_cast<uintptr_t>(p) << 8) | e} {
+    assert(ptr() == p);
+  }
+
+  /* implicit */ TaggedPtr(std::nullptr_t) noexcept : raw_{0} {}
+
+  TaggedPtr& operator=(std::nullptr_t) noexcept {
+    raw_ = 0;
+    return *this;
+  }
+
+  T& operator*() const noexcept {
+    return *ptr();
+  }
+
+  T* operator->() const noexcept {
+    return std::addressof(*ptr());
+  }
+
+  T* ptr() const {
+    return reinterpret_cast<T*>(raw_ >> 8);
+  }
+
+  void setPtr(T* p) {
+    *this = TaggedPtr{p, extra()};
+    assert(ptr() == p);
+  }
+
+  uint8_t extra() const {
+    return static_cast<uint8_t>(raw_);
+  }
+
+  void setExtra(uint8_t e) {
+    *this = TaggedPtr{ptr(), e};
+  }
+
+  bool operator==(TaggedPtr const& rhs) const {
+    return raw_ == rhs.raw_;
+  }
+  bool operator!=(TaggedPtr const& rhs) const {
+    return !(*this == rhs);
+  }
+
+  bool operator<(TaggedPtr const& rhs) const noexcept {
+    return raw_ < rhs.raw_;
+  }
+
+  bool operator==(std::nullptr_t) const noexcept {
+    return raw_ == 0;
+  }
+  bool operator!=(std::nullptr_t) const noexcept {
+    return !(*this == nullptr);
+  }
+
+ private:
+  // TODO: verify no high-bit extension needed on aarch64
+  uintptr_t raw_;
+};
+
+#endif // FOLLY_X64 || FOLLY_AARCH64
+
+} // namespace detail
+} // namespace f14
+} // namespace folly

folly/container/detail/F14Table.cpp

+/*
+ * Copyright 2017-present Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <folly/container/detail/F14Table.h>
+
+///////////////////////////////////
+#if FOLLY_F14_VECTOR_INTRINSICS_AVAILABLE
+///////////////////////////////////
+
+namespace folly {
+namespace f14 {
+namespace detail {
+
+__m128i kEmptyTagVector = {};
+
+} // namespace detail
+} // namespace f14
+} // namespace folly
+
+///////////////////////////////////
+#endif // FOLLY_F14_VECTOR_INTRINSICS_AVAILABLE
+///////////////////////////////////