Use sized deallocation in SysAllocator and Arena (2nd try)

Summary:
Use sized-deallocation (`sdallocx`) if possible in `folly::SysAllocator` and `folly::Arena`.

`Arena` has always allocated two types of blocks:
1. Normal (fixed-sized): size is the "goodSize adjusted" `minBlockSize`
2. Large (variable-sized): when #1 is too small

Type #2 makes sized-deallocation tricky -- we need somewhere to remember the allocated sizes.

The old code used a single type `Block` and kept a single list. Here I change to have two types and two lists.  The `LargeBlock` has an additional `allocSize` data member.

This makes the Arena object itself 16B larger, but seems better than adding a 4B `allocSize` to each and every block, regardless of type.

Note that, prior to this change, it was possible to `merge()` arenas with different `minBlockSize`.  This is no longer possible.

Reviewed By: davidtgoldblatt

Differential Revision: D22466906

fbshipit-source-id: 719f357a0a1f6cfcda3208391837195c3859ab69

folly/Memory.h

@@ -33,6 +33,7 @@
 #include <folly/functional/Invoke.h>
 #include <folly/lang/Align.h>
 #include <folly/lang/Exception.h>
+#include <folly/memory/Malloc.h>
 #include <folly/portability/Config.h>
 #include <folly/portability/Malloc.h>
 
@@ -412,8 +413,9 @@ class SysAllocator {
     }
     return static_cast<T*>(p);
   }
-  void deallocate(T* p, size_t /* count */) {
-    std::free(p);
+  void deallocate(T* p, size_t count) {
+    using lifted = typename detail::lift_void_to_char<T>::type;
+    sizedFree(p, count * sizeof(lifted));
   }
 
   friend bool operator==(Self const&, Self const&) noexcept {

folly/memory/Arena-inl.h

@@ -22,27 +22,8 @@
 
 namespace folly {
 
-template <class Alloc>
-std::pair<typename Arena<Alloc>::Block*, size_t>
-Arena<Alloc>::Block::allocate(Alloc& alloc, size_t size, bool allowSlack) {
-  size_t allocSize = sizeof(Block) + size;
-  if (allowSlack) {
-    allocSize = ArenaAllocatorTraits<Alloc>::goodSize(alloc, allocSize);
-  }
-
-  void* mem = std::allocator_traits<Alloc>::allocate(alloc, allocSize);
-  return std::make_pair(new (mem) Block(), allocSize - sizeof(Block));
-}
-
-template <class Alloc>
-void Arena<Alloc>::Block::deallocate(Alloc& alloc) {
-  this->~Block();
-  std::allocator_traits<Alloc>::deallocate(alloc, this, 1);
-}
-
 template <class Alloc>
 void* Arena<Alloc>::allocateSlow(size_t size) {
-  std::pair<Block*, size_t> p;
   char* start;
 
   size_t allocSize;
@@ -55,31 +36,41 @@ void* Arena<Alloc>::allocateSlow(size_t size) {
   }
 
   if (size > minBlockSize()) {
-    // Allocate a large block for this chunk only, put it at the back of the
-    // list so it doesn't get used for small allocations; don't change ptr_
-    // and end_, let them point into a normal block (or none, if they're
-    // null)
-    p = Block::allocate(alloc(), size, false);
-    start = p.first->start();
-    blocks_.push_back(*p.first);
+    // Allocate a large block for this chunk only; don't change ptr_ and end_,
+    // let them point into a normal block (or none, if they're null)
+    allocSize = sizeof(LargeBlock) + size;
+    void* mem = AllocTraits::allocate(alloc(), allocSize);
+    auto blk = new (mem) LargeBlock(allocSize);
+    start = blk->start();
+    largeBlocks_.push_front(*blk);
   } else {
     // Allocate a normal sized block and carve out size bytes from it
-    p = Block::allocate(alloc(), minBlockSize(), true);
-    start = p.first->start();
-    blocks_.push_front(*p.first);
+    // Will allocate more than size bytes if convenient
+    // (via ArenaAllocatorTraits::goodSize()) as we'll try to pack small
+    // allocations in this block.
+    allocSize = blockGoodAllocSize();
+    void* mem = AllocTraits::allocate(alloc(), allocSize);
+    auto blk = new (mem) Block();
+    start = blk->start();
+    blocks_.push_front(*blk);
     ptr_ = start + size;
-    end_ = start + p.second;
+    end_ = start + allocSize - sizeof(Block);
+    assert(ptr_ <= end_);
   }
 
-  assert(p.second >= size);
-  totalAllocatedSize_ += p.second + sizeof(Block);
+  totalAllocatedSize_ += allocSize;
   return start;
 }
 
 template <class Alloc>
 void Arena<Alloc>::merge(Arena<Alloc>&& other) {
+  FOLLY_SAFE_CHECK(
+      blockGoodAllocSize() == other.blockGoodAllocSize(),
+      "cannot merge arenas of different minBlockSize");
   blocks_.splice_after(blocks_.before_begin(), other.blocks_);
   other.blocks_.clear();
+  largeBlocks_.splice_after(largeBlocks_.before_begin(), other.largeBlocks_);
+  other.largeBlocks_.clear();
   other.ptr_ = other.end_ = nullptr;
   totalAllocatedSize_ += other.totalAllocatedSize_;
   other.totalAllocatedSize_ = 0;
@@ -87,7 +78,15 @@ void Arena<Alloc>::merge(Arena<Alloc>&& other) {
 
 template <class Alloc>
 Arena<Alloc>::~Arena() {
-  blocks_.clear_and_dispose([this](Block* b) { b->deallocate(this->alloc()); });
+  blocks_.clear_and_dispose([this](Block* b) {
+    b->~Block();
+    AllocTraits::deallocate(alloc(), b, blockGoodAllocSize());
+  });
+  largeBlocks_.clear_and_dispose([this](LargeBlock* b) {
+    auto size = b->allocSize;
+    b->~LargeBlock();
+    AllocTraits::deallocate(alloc(), b, size);
+  });
 }
 
 } // namespace folly

folly/memory/Arena.h

@@ -126,30 +126,37 @@ class Arena {
   Arena& operator=(Arena&&) = delete;
 
  private:
-  struct Block;
-  typedef boost::intrusive::slist_member_hook<boost::intrusive::tag<Arena>>
-      BlockLink;
+  using AllocTraits = std::allocator_traits<Alloc>;
+  using BlockLink = boost::intrusive::slist_member_hook<>;
 
   struct alignas(max_align_v) Block {
     BlockLink link;
 
-    // Allocate a block with at least size bytes of storage.
-    // If allowSlack is true, allocate more than size bytes if convenient
-    // (via ArenaAllocatorTraits::goodSize()) as we'll try to pack small
-    // allocations in this block.
-    static std::pair<Block*, size_t>
-    allocate(Alloc& alloc, size_t size, bool allowSlack);
-    void deallocate(Alloc& alloc);
-
     char* start() {
       return reinterpret_cast<char*>(this + 1);
     }
 
-   private:
     Block() = default;
     ~Block() = default;
   };
 
+  constexpr size_t blockGoodAllocSize() {
+    return ArenaAllocatorTraits<Alloc>::goodSize(
+        alloc(), sizeof(Block) + minBlockSize());
+  }
+
+  struct alignas(max_align_v) LargeBlock {
+    BlockLink link;
+    const size_t allocSize;
+
+    char* start() {
+      return reinterpret_cast<char*>(this + 1);
+    }
+
+    LargeBlock(size_t s) : allocSize(s) {}
+    ~LargeBlock() = default;
+  };
+
  public:
   static constexpr size_t kDefaultMinBlockSize = 4096 - sizeof(Block);
   static constexpr size_t kNoSizeLimit = 0;
@@ -183,6 +190,13 @@ class Arena {
       boost::intrusive::cache_last<true>>
       BlockList;
 
+  typedef boost::intrusive::slist<
+      LargeBlock,
+      boost::intrusive::member_hook<LargeBlock, BlockLink, &LargeBlock::link>,
+      boost::intrusive::constant_time_size<false>,
+      boost::intrusive::cache_last<true>>
+      LargeBlockList;
+
   void* allocateSlow(size_t size);
 
   // Empty member optimization: package Alloc with a non-empty member
@@ -206,6 +220,7 @@ class Arena {
 
   AllocAndSize allocAndSize_;
   BlockList blocks_;
+  LargeBlockList largeBlocks_;
   char* ptr_;
   char* end_;
   size_t totalAllocatedSize_;

folly/memory/ThreadCachedArena.cpp

@@ -21,7 +21,9 @@
 namespace folly {
 
 ThreadCachedArena::ThreadCachedArena(size_t minBlockSize, size_t maxAlign)
-    : minBlockSize_(minBlockSize), maxAlign_(maxAlign) {}
+    : minBlockSize_(minBlockSize),
+      maxAlign_(maxAlign),
+      zombies_(folly::in_place, minBlockSize) {}
 
 SysArena* ThreadCachedArena::allocateThreadLocalArena() {
   auto arena = new SysArena(minBlockSize_, SysArena::kNoSizeLimit, maxAlign_);