Skip to content

F
folly
Commit 2ff9d181 authored by Giuseppe Ottaviano's avatar Giuseppe Ottaviano Committed by Facebook Github Bot
Browse files
Options

Outline several fbstring/malloc functions 

Summary: Outline all the non-trivial constructor and the destructor, `assign()`, and `append()` to reduce code size. The fast path of copy, and moves, are instead still inlined.

Reviewed By: philippv

Differential Revision: D3896612

fbshipit-source-id: 25050d4ba28d25da226a7bc49d5b542947d0c512
parent 471f03d0
Hide whitespace changes
Inline Side-by-side
Showing with 87 additions and 70 deletions
folly/FBString.h
View file @ 2ff9d181
......@@ -362,15 +362,10 @@ public:
}
~fbstring_core() noexcept {
auto const c = category();
if (c == Category::isSmall) {
return;
}
if (c == Category::isMedium) {
free(ml_.data_);
if (category() == Category::isSmall) {
return;
}
RefCounted::decrementRefs(ml_.data_);
destroyMediumLarge();
}
// Snatches a previously mallocated string. The parameter "size"
......@@ -444,6 +439,7 @@ public:
}
}
FOLLY_MALLOC_NOINLINE
void reserve(size_t minCapacity, bool disableSSO = FBSTRING_DISABLE_SSO) {
switch (category()) {
case Category::isSmall:
......@@ -514,6 +510,16 @@ private:
setSmallSize(0);
}
FOLLY_MALLOC_NOINLINE void destroyMediumLarge() noexcept {
auto const c = category();
FBSTRING_ASSERT(c != Category::isSmall);
if (c == Category::isMedium) {
free(ml_.data_);
} else {
RefCounted::decrementRefs(ml_.data_);
}
}
struct RefCounted {
std::atomic<size_t> refCount_;
Char data_[1];
......@@ -665,6 +671,7 @@ private:
void shrinkMedium(size_t delta);
void shrinkLarge(size_t delta);
void unshare(size_t minCapacity = 0);
Char* mutableDataLarge();
};
......@@ -688,7 +695,8 @@ inline void fbstring_core<Char>::copySmall(const fbstring_core& rhs) {
}
template <class Char>
inline void fbstring_core<Char>::copyMedium(const fbstring_core& rhs) {
FOLLY_MALLOC_NOINLINE inline void fbstring_core<Char>::copyMedium(
const fbstring_core& rhs) {
// Medium strings are copied eagerly. Don't forget to allocate
// one extra Char for the null terminator.
auto const allocSize = goodMallocSize((1 + rhs.ml_.size_) * sizeof(Char));
......@@ -702,7 +710,8 @@ inline void fbstring_core<Char>::copyMedium(const fbstring_core& rhs) {
}
template <class Char>
inline void fbstring_core<Char>::copyLarge(const fbstring_core& rhs) {
FOLLY_MALLOC_NOINLINE inline void fbstring_core<Char>::copyLarge(
const fbstring_core& rhs) {
// Large strings are just refcounted
ml_ = rhs.ml_;
RefCounted::incrementRefs(ml_.data_);
......@@ -754,7 +763,7 @@ inline void fbstring_core<Char>::initSmall(
}
template <class Char>
inline void fbstring_core<Char>::initMedium(
FOLLY_MALLOC_NOINLINE inline void fbstring_core<Char>::initMedium(
const Char* const data, const size_t size) {
// Medium strings are allocated normally. Don't forget to
// allocate one extra Char for the terminating null.
......@@ -767,7 +776,7 @@ inline void fbstring_core<Char>::initMedium(
}
template <class Char>
inline void fbstring_core<Char>::initLarge(
FOLLY_MALLOC_NOINLINE inline void fbstring_core<Char>::initLarge(
const Char* const data, const size_t size) {
// Large strings are allocated differently
size_t effectiveCapacity = size;
......@@ -778,44 +787,43 @@ inline void fbstring_core<Char>::initLarge(
ml_.data_[size] = '\0';
}
template <class Char>
FOLLY_MALLOC_NOINLINE inline void fbstring_core<Char>::unshare(
size_t minCapacity) {
FBSTRING_ASSERT(category() == Category::isLarge);
size_t effectiveCapacity = std::max(minCapacity, ml_.capacity());
auto const newRC = RefCounted::create(&effectiveCapacity);
// If this fails, someone placed the wrong capacity in an
// fbstring.
FBSTRING_ASSERT(effectiveCapacity >= ml_.capacity());
// Also copies terminator.
fbstring_detail::podCopy(ml_.data_, ml_.data_ + ml_.size_ + 1, newRC->data_);
RefCounted::decrementRefs(ml_.data_);
ml_.data_ = newRC->data_;
ml_.setCapacity(effectiveCapacity, Category::isLarge);
// size_ remains unchanged.
}
template <class Char>
inline Char* fbstring_core<Char>::mutableDataLarge() {
FBSTRING_ASSERT(category() == Category::isLarge);
if (RefCounted::refs(ml_.data_) > 1) {
// Ensure unique.
size_t effectiveCapacity = ml_.capacity();
auto const newRC = RefCounted::create(&effectiveCapacity);
// If this fails, someone placed the wrong capacity in an
// fbstring.
FBSTRING_ASSERT(effectiveCapacity >= ml_.capacity());
// Also copies terminator.
fbstring_detail::podCopy(
ml_.data_, ml_.data_ + ml_.size_ + 1, newRC->data_);
RefCounted::decrementRefs(ml_.data_);
ml_.data_ = newRC->data_;
if (RefCounted::refs(ml_.data_) > 1) { // Ensure unique.
unshare();
}
return ml_.data_;
}
template <class Char>
inline void fbstring_core<Char>::reserveLarge(size_t minCapacity) {
FOLLY_MALLOC_NOINLINE inline void fbstring_core<Char>::reserveLarge(
size_t minCapacity) {
FBSTRING_ASSERT(category() == Category::isLarge);
// Ensure unique
if (RefCounted::refs(ml_.data_) > 1) {
if (RefCounted::refs(ml_.data_) > 1) { // Ensure unique
// We must make it unique regardless; in-place reallocation is
// useless if the string is shared. In order to not surprise
// people, reserve the new block at current capacity or
// more. That way, a string's capacity never shrinks after a
// call to reserve.
minCapacity = std::max(minCapacity, ml_.capacity());
auto const newRC = RefCounted::create(&minCapacity);
// Also copies terminator.
fbstring_detail::podCopy(
ml_.data_, ml_.data_ + ml_.size_ + 1, newRC->data_);
RefCounted::decrementRefs(ml_.data_);
ml_.data_ = newRC->data_;
ml_.setCapacity(minCapacity, Category::isLarge);
// size remains unchanged
unshare(minCapacity);
} else {
// String is not shared, so let's try to realloc (if needed)
if (minCapacity > ml_.capacity()) {
......@@ -830,7 +838,8 @@ inline void fbstring_core<Char>::reserveLarge(size_t minCapacity) {
}
template <class Char>
inline void fbstring_core<Char>::reserveMedium(const size_t minCapacity) {
FOLLY_MALLOC_NOINLINE inline void fbstring_core<Char>::reserveMedium(
const size_t minCapacity) {
FBSTRING_ASSERT(category() == Category::isMedium);
// String is not shared
if (minCapacity <= ml_.capacity()) {
......@@ -862,7 +871,7 @@ inline void fbstring_core<Char>::reserveMedium(const size_t minCapacity) {
}
template <class Char>
inline void fbstring_core<Char>::reserveSmall(
FOLLY_MALLOC_NOINLINE inline void fbstring_core<Char>::reserveSmall(
size_t minCapacity, const bool disableSSO) {
FBSTRING_ASSERT(category() == Category::isSmall);
if (!disableSSO && minCapacity <= maxSmallSize) {
......@@ -1140,21 +1149,23 @@ public:
assign(str, pos, n);
}
FOLLY_MALLOC_NOINLINE
/* implicit */ basic_fbstring(const value_type* s, const A& /*a*/ = A())
: store_(s, basic_fbstring::traitsLength(s)) {
}
: store_(s, traitsLength(s)) {}
FOLLY_MALLOC_NOINLINE
basic_fbstring(const value_type* s, size_type n, const A& /*a*/ = A())
: store_(s, n) {
}
FOLLY_MALLOC_NOINLINE
basic_fbstring(size_type n, value_type c, const A& /*a*/ = A()) {
auto const pData = store_.expandNoinit(n);
fbstring_detail::podFill(pData, pData + n, c);
}
template <class InIt>
basic_fbstring(
FOLLY_MALLOC_NOINLINE basic_fbstring(
InIt begin,
InIt end,
typename std::enable_if<
......@@ -1164,6 +1175,7 @@ public:
}
// Specialization for const char*, const char*
FOLLY_MALLOC_NOINLINE
basic_fbstring(const value_type* b, const value_type* e, const A& /*a*/ = A())
: store_(b, e - b) {
}
......@@ -1175,12 +1187,12 @@ public:
}
// Construction from initialization list
FOLLY_MALLOC_NOINLINE
basic_fbstring(std::initializer_list<value_type> il) {
assign(il.begin(), il.end());
}
~basic_fbstring() noexcept {
}
~basic_fbstring() noexcept {}
basic_fbstring& operator=(const basic_fbstring& lhs);
......@@ -1352,7 +1364,7 @@ public:
basic_fbstring& append(const value_type* s, size_type n);
basic_fbstring& append(const value_type* s) {
return append(s, traits_type::length(s));
return append(s, traitsLength(s));
}
basic_fbstring& append(size_type n, value_type c);
......@@ -1386,7 +1398,7 @@ public:
basic_fbstring& assign(const value_type* s, const size_type n);
basic_fbstring& assign(const value_type* s) {
return assign(s, traits_type::length(s));
return assign(s, traitsLength(s));
}
basic_fbstring& assign(std::initializer_list<value_type> il) {
......@@ -1416,7 +1428,7 @@ public:
}
basic_fbstring& insert(size_type pos, const value_type* s) {
return insert(pos, s, traits_type::length(s));
return insert(pos, s, traitsLength(s));
}
basic_fbstring& insert(size_type pos, size_type n, value_type c) {
......@@ -1525,7 +1537,7 @@ public:
// Replaces at most n1 chars of *this, starting with pos, with chars from s
basic_fbstring& replace(size_type pos, size_type n1, const value_type* s) {
return replace(pos, n1, s, traits_type::length(s));
return replace(pos, n1, s, traitsLength(s));
}
// Replaces at most n1 chars of *this, starting with pos, with n2
......@@ -1551,7 +1563,7 @@ public:
}
basic_fbstring& replace(iterator i1, iterator i2, const value_type* s) {
return replace(i1, i2, s, traits_type::length(s));
return replace(i1, i2, s, traitsLength(s));
}
private:
......@@ -1654,7 +1666,7 @@ private:
const;
size_type find(const value_type* s, size_type pos = 0) const {
return find(s, pos, traits_type::length(s));
return find(s, pos, traitsLength(s));
}
size_type find (value_type c, size_type pos = 0) const {
......@@ -1668,7 +1680,7 @@ private:
size_type rfind(const value_type* s, size_type pos, size_type n) const;
size_type rfind(const value_type* s, size_type pos = npos) const {
return rfind(s, pos, traits_type::length(s));
return rfind(s, pos, traitsLength(s));
}
size_type rfind(value_type c, size_type pos = npos) const {
......@@ -1683,7 +1695,7 @@ private:
const;
size_type find_first_of(const value_type* s, size_type pos = 0) const {
return find_first_of(s, pos, traits_type::length(s));
return find_first_of(s, pos, traitsLength(s));
}
size_type find_first_of(value_type c, size_type pos = 0) const {
......@@ -1699,7 +1711,7 @@ private:
size_type find_last_of (const value_type* s,
size_type pos = npos) const {
return find_last_of(s, pos, traits_type::length(s));
return find_last_of(s, pos, traitsLength(s));
}
size_type find_last_of (value_type c, size_type pos = npos) const {
......@@ -1716,7 +1728,7 @@ private:
size_type find_first_not_of(const value_type* s,
size_type pos = 0) const {
return find_first_not_of(s, pos, traits_type::length(s));
return find_first_not_of(s, pos, traitsLength(s));
}
size_type find_first_not_of(value_type c, size_type pos = 0) const {
......@@ -1733,7 +1745,7 @@ private:
size_type find_last_not_of(const value_type* s,
size_type pos = npos) const {
return find_last_not_of(s, pos, traits_type::length(s));
return find_last_not_of(s, pos, traitsLength(s));
}
size_type find_last_not_of (value_type c, size_type pos = npos) const {
......@@ -1766,7 +1778,7 @@ private:
int compare(size_type pos1, size_type n1,
const value_type* s) const {
return compare(pos1, n1, s, traits_type::length(s));
return compare(pos1, n1, s, traitsLength(s));
}
int compare(size_type pos1, size_type n1,
......@@ -1788,9 +1800,9 @@ private:
// Code from Jean-Francois Bastien (03/26/2007)
int compare(const value_type* s) const {
// Could forward to compare(0, size(), s, traits_type::length(s))
// Could forward to compare(0, size(), s, traitsLength(s))
// but that does two extra checks
const size_type n1(size()), n2(traits_type::length(s));
const size_type n1(size()), n2(traitsLength(s));
const int r = traits_type::compare(data(), s, std::min(n1, n2));
return r != 0 ? r : n1 > n2 ? 1 : n1 < n2 ? -1 : 0;
}
......@@ -1801,7 +1813,7 @@ private:
};
template <typename E, class T, class A, class S>
inline typename basic_fbstring<E, T, A, S>::size_type
FOLLY_MALLOC_NOINLINE inline typename basic_fbstring<E, T, A, S>::size_type
basic_fbstring<E, T, A, S>::traitsLength(const value_type* s) {
return s ? traits_type::length(s)
: (std::__throw_logic_error(
......@@ -1891,8 +1903,8 @@ inline basic_fbstring<E, T, A, S>& basic_fbstring<E, T, A, S>::append(
}
template <typename E, class T, class A, class S>
inline basic_fbstring<E, T, A, S>& basic_fbstring<E, T, A, S>::append(
const value_type* s, size_type n) {
FOLLY_MALLOC_NOINLINE inline basic_fbstring<E, T, A, S>&
basic_fbstring<E, T, A, S>::append(const value_type* s, size_type n) {
Invariant checker(*this);
if (FBSTRING_UNLIKELY(!n)) {
......@@ -1942,11 +1954,10 @@ inline basic_fbstring<E, T, A, S>& basic_fbstring<E, T, A, S>::assign(
}
template <typename E, class T, class A, class S>
inline basic_fbstring<E, T, A, S>& basic_fbstring<E, T, A, S>::assign(
const value_type* s, const size_type n) {
FOLLY_MALLOC_NOINLINE inline basic_fbstring<E, T, A, S>&
basic_fbstring<E, T, A, S>::assign(const value_type* s, const size_type n) {
Invariant checker(*this);
// s can alias this, we need to use podMove.
if (n == 0) {
resize(0);
} else if (size() >= n) {
......
folly/Malloc.h
View file @ 2ff9d181
......@@ -111,6 +111,7 @@ __attribute__((__weak__));
#include <cstdlib>
#include <cstring>
#include <atomic>
#include <new>
#ifdef _LIBSTDCXX_FBSTRING
......@@ -121,17 +122,21 @@ namespace folly {
#endif
// Cannot depend on Portability.h when _LIBSTDCXX_FBSTRING.
// Disabled for nvcc because it fails on attributes on lambdas.
#if defined(__GNUC__) && !defined(__NVCC__)
#if defined(__GNUC__)
#define FOLLY_MALLOC_NOINLINE __attribute__((__noinline__))
// This is for checked malloc-like functions (returns non-null pointer
// which cannot alias any outstanding pointer).
#define FOLLY_MALLOC_CHECKED_MALLOC \
__attribute__((__returns_nonnull__, __malloc__))
#else
#define FOLLY_MALLOC_NOINLINE
#define FOLLY_MALLOC_CHECKED_MALLOC
#endif
/**
* Determine if we are using jemalloc or not.
*/
inline bool usingJEMalloc() noexcept {
FOLLY_MALLOC_NOINLINE inline bool usingJEMalloc() noexcept {
// Checking for rallocx != NULL is not sufficient; we may be in a dlopen()ed
// module that depends on libjemalloc, so rallocx is resolved, but the main
// program might be using a different memory allocator.
......@@ -140,7 +145,7 @@ inline bool usingJEMalloc() noexcept {
// per-thread counter of allocated memory increases. This makes me
// feel dirty inside. Also note that this requires jemalloc to have
// been compiled with --enable-stats.
static const bool result = [] () FOLLY_MALLOC_NOINLINE noexcept {
static const bool result = [] () noexcept {
// Some platforms (*cough* OSX *cough*) require weak symbol checks to be
// in the form if (mallctl != nullptr). Not if (mallctl) or if (!mallctl)
// (!!). http://goo.gl/xpmctm
......@@ -235,10 +240,11 @@ inline void* checkedRealloc(void* ptr, size_t size) {
* routine just tries to call realloc() (thus benefitting of potential
* copy-free coalescing) unless there's too much slack memory.
*/
inline void* smartRealloc(void* p,
const size_t currentSize,
const size_t currentCapacity,
const size_t newCapacity) {
FOLLY_MALLOC_CHECKED_MALLOC FOLLY_MALLOC_NOINLINE inline void* smartRealloc(
void* p,
const size_t currentSize,
const size_t currentCapacity,
const size_t newCapacity) {
assert(p);
assert(currentSize <= currentCapacity &&
currentCapacity < newCapacity);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment