Skip to content

F
folly
Commit d60894bf authored by Yedidya Feldblum's avatar Yedidya Feldblum Committed by Facebook GitHub Bot
Browse files
Options

condense the atomic-fetch-bit-op implementations 

Summary: Coalesce the inline-asm. Use invokers.

Reviewed By: Gownta

Differential Revision: D32302511

fbshipit-source-id: 6edf75c6ef820ed80e751d965174e5acfaac957e
parent 80c05933
Hide whitespace changes
Inline Side-by-side
Showing with 188 additions and 228 deletions
folly/synchronization/AtomicUtil-inl.h
View file @ d60894bf
...@@ -22,8 +22,6 @@ ...@@ -22,8 +22,6 @@
#include <tuple> #include <tuple>
#include <type_traits> #include <type_traits>
#include <folly/Portability.h>
#ifdef _WIN32 #ifdef _WIN32
#include <intrin.h> #include <intrin.h>
#endif #endif
...@@ -111,29 +109,41 @@ namespace detail { ...@@ -111,29 +109,41 @@ namespace detail {
// Currently, at the time of writing it seems like gcc7 and greater can make // Currently, at the time of writing it seems like gcc7 and greater can make
// this optimization and clang cannot - https://gcc.godbolt.org/z/Q83rxX // this optimization and clang cannot - https://gcc.godbolt.org/z/Q83rxX
template <typename Atomic> struct atomic_fetch_set_fallback_fn {
bool atomic_fetch_set_fallback( template <typename Atomic>
Atomic& atomic, std::size_t bit, std::memory_order order) { bool operator()(
using Integer = decltype(atomic.load()); Atomic& atomic, std::size_t bit, std::memory_order order) const {
auto mask = Integer(Integer{0b1} << bit); using Integer = decltype(atomic.load());
return (atomic.fetch_or(mask, order) & mask); auto mask = Integer(Integer{0b1} << bit);
} return (atomic.fetch_or(mask, order) & mask);
}
template <typename Atomic> };
bool atomic_fetch_reset_fallback( FOLLY_INLINE_VARIABLE constexpr atomic_fetch_set_fallback_fn
Atomic& atomic, std::size_t bit, std::memory_order order) { atomic_fetch_set_fallback{};
using Integer = decltype(atomic.load());
auto mask = Integer(Integer{0b1} << bit); struct atomic_fetch_reset_fallback_fn {
return (atomic.fetch_and(Integer(~mask), order) & mask); template <typename Atomic>
} bool operator()(
Atomic& atomic, std::size_t bit, std::memory_order order) const {
template <typename Atomic> using Integer = decltype(atomic.load());
bool atomic_fetch_flip_fallback( auto mask = Integer(Integer{0b1} << bit);
Atomic& atomic, std::size_t bit, std::memory_order order) { return (atomic.fetch_and(Integer(~mask), order) & mask);
using Integer = decltype(atomic.load()); }
auto mask = Integer(Integer{0b1} << bit); };
return (atomic.fetch_xor(mask, order) & mask); FOLLY_INLINE_VARIABLE constexpr atomic_fetch_reset_fallback_fn
} atomic_fetch_reset_fallback{};
struct atomic_fetch_flip_fallback_fn {
template <typename Atomic>
bool operator()(
Atomic& atomic, std::size_t bit, std::memory_order order) const {
using Integer = decltype(atomic.load());
auto mask = Integer(Integer{0b1} << bit);
return (atomic.fetch_xor(mask, order) & mask);
}
};
FOLLY_INLINE_VARIABLE constexpr atomic_fetch_flip_fallback_fn
atomic_fetch_flip_fallback{};
/** /**
* A simple trait to determine if the given type is an instantiation of * A simple trait to determine if the given type is an instantiation of
...@@ -214,35 +224,64 @@ inline bool atomic_fetch_flip_native( ...@@ -214,35 +224,64 @@ inline bool atomic_fetch_flip_native(
#else #else
#define FOLLY_DETAIL_ATOMIC_BIT_OP_DEFINE(instr) \
struct atomic_fetch_bit_op_native_##instr##_fn { \
template <typename I> \
FOLLY_ERASE bool operator()(I* ptr, I bit) const { \
bool out = false; \
if (sizeof(I) == 2) { \
asm volatile("lock " #instr "w %1, (%2); setc %0" \
: "=r"(out) \
: "ri"(bit), "r"(ptr) \
: "memory", "flags"); \
} \
if (sizeof(I) == 4) { \
asm volatile("lock " #instr "l %1, (%2); setc %0" \
: "=r"(out) \
: "ri"(bit), "r"(ptr) \
: "memory", "flags"); \
} \
if (sizeof(I) == 8) { \
asm volatile("lock " #instr "q %1, (%2); setc %0" \
: "=r"(out) \
: "ri"(bit), "r"(ptr) \
: "memory", "flags"); \
} \
return out; \
} \
}; \
FOLLY_INLINE_VARIABLE constexpr atomic_fetch_bit_op_native_##instr##_fn \
atomic_fetch_bit_op_native_##instr
FOLLY_DETAIL_ATOMIC_BIT_OP_DEFINE(bts);
FOLLY_DETAIL_ATOMIC_BIT_OP_DEFINE(btr);
FOLLY_DETAIL_ATOMIC_BIT_OP_DEFINE(btc);
#undef FOLLY_DETAIL_ATOMIC_BIT_OP_DEFINE
template <typename Integer, typename Op, typename Fb>
FOLLY_ERASE bool atomic_fetch_bit_op_native_(
std::atomic<Integer>& atomic,
std::size_t bit,
std::memory_order order,
Op op,
Fb fb) {
constexpr auto atomic_size = sizeof(Integer);
constexpr auto lo_size = std::size_t(2);
constexpr auto hi_size = std::size_t(8);
// some versions of TSAN do not properly instrument the inline assembly
if (atomic_size < lo_size || atomic_size > hi_size || folly::kIsSanitize) {
return fb(atomic, bit, order);
}
return op(reinterpret_cast<Integer*>(&atomic), Integer(bit));
}
template <typename Integer> template <typename Integer>
inline bool atomic_fetch_set_native( inline bool atomic_fetch_set_native(
std::atomic<Integer>& atomic, std::size_t bit, std::memory_order order) { std::atomic<Integer>& atomic, std::size_t bit, std::memory_order order) {
auto previous = false; auto op = atomic_fetch_bit_op_native_bts;
auto fb = atomic_fetch_set_fallback;
if /* constexpr */ (sizeof(Integer) == 2) { return atomic_fetch_bit_op_native_(atomic, bit, order, op, fb);
auto pointer = reinterpret_cast<std::uint16_t*>(&atomic);
asm volatile("lock; btsw %1, (%2); setc %0"
: "=r"(previous)
: "ri"(static_cast<std::uint16_t>(bit)), "r"(pointer)
: "memory", "flags");
} else if /* constexpr */ (sizeof(Integer) == 4) {
auto pointer = reinterpret_cast<std::uint32_t*>(&atomic);
asm volatile("lock; btsl %1, (%2); setc %0"
: "=r"(previous)
: "ri"(static_cast<std::uint32_t>(bit)), "r"(pointer)
: "memory", "flags");
} else if /* constexpr */ (sizeof(Integer) == 8) {
auto pointer = reinterpret_cast<std::uint64_t*>(&atomic);
asm volatile("lock; btsq %1, (%2); setc %0"
: "=r"(previous)
: "ri"(static_cast<std::uint64_t>(bit)), "r"(pointer)
: "memory", "flags");
} else {
assert(sizeof(Integer) == 1);
return atomic_fetch_set_fallback(atomic, bit, order);
}
return previous;
} }
template <typename Atomic> template <typename Atomic>
...@@ -255,32 +294,9 @@ inline bool atomic_fetch_set_native( ...@@ -255,32 +294,9 @@ inline bool atomic_fetch_set_native(
template <typename Integer> template <typename Integer>
inline bool atomic_fetch_reset_native( inline bool atomic_fetch_reset_native(
std::atomic<Integer>& atomic, std::size_t bit, std::memory_order order) { std::atomic<Integer>& atomic, std::size_t bit, std::memory_order order) {
auto previous = false; auto op = atomic_fetch_bit_op_native_btr;
auto fb = atomic_fetch_reset_fallback;
if /* constexpr */ (sizeof(Integer) == 2) { return atomic_fetch_bit_op_native_(atomic, bit, order, op, fb);
auto pointer = reinterpret_cast<std::uint16_t*>(&atomic);
asm volatile("lock; btrw %1, (%2); setc %0"
: "=r"(previous)
: "ri"(static_cast<std::uint16_t>(bit)), "r"(pointer)
: "memory", "flags");
} else if /* constexpr */ (sizeof(Integer) == 4) {
auto pointer = reinterpret_cast<std::uint32_t*>(&atomic);
asm volatile("lock; btrl %1, (%2); setc %0"
: "=r"(previous)
: "ri"(static_cast<std::uint32_t>(bit)), "r"(pointer)
: "memory", "flags");
} else if /* constexpr */ (sizeof(Integer) == 8) {
auto pointer = reinterpret_cast<std::uint64_t*>(&atomic);
asm volatile("lock; btrq %1, (%2); setc %0"
: "=r"(previous)
: "ri"(static_cast<std::uint64_t>(bit)), "r"(pointer)
: "memory", "flags");
} else {
assert(sizeof(Integer) == 1);
return atomic_fetch_reset_fallback(atomic, bit, order);
}
return previous;
} }
template <typename Atomic> template <typename Atomic>
...@@ -293,32 +309,9 @@ bool atomic_fetch_reset_native( ...@@ -293,32 +309,9 @@ bool atomic_fetch_reset_native(
template <typename Integer> template <typename Integer>
inline bool atomic_fetch_flip_native( inline bool atomic_fetch_flip_native(
std::atomic<Integer>& atomic, std::size_t bit, std::memory_order order) { std::atomic<Integer>& atomic, std::size_t bit, std::memory_order order) {
auto previous = false; auto op = atomic_fetch_bit_op_native_btc;
auto fb = atomic_fetch_flip_fallback;
if /* constexpr */ (sizeof(Integer) == 2) { return atomic_fetch_bit_op_native_(atomic, bit, order, op, fb);
auto pointer = reinterpret_cast<std::uint16_t*>(&atomic);
asm volatile("lock; btcw %1, (%2); setc %0"
: "=r"(previous)
: "ri"(static_cast<std::uint16_t>(bit)), "r"(pointer)
: "memory", "flags");
} else if /* constexpr */ (sizeof(Integer) == 4) {
auto pointer = reinterpret_cast<std::uint32_t*>(&atomic);
asm volatile("lock; btcl %1, (%2); setc %0"
: "=r"(previous)
: "ri"(static_cast<std::uint32_t>(bit)), "r"(pointer)
: "memory", "flags");
} else if /* constexpr */ (sizeof(Integer) == 8) {
auto pointer = reinterpret_cast<std::uint64_t*>(&atomic);
asm volatile("lock; btcq %1, (%2); setc %0"
: "=r"(previous)
: "ri"(static_cast<std::uint64_t>(bit)), "r"(pointer)
: "memory", "flags");
} else {
assert(sizeof(Integer) == 1);
return atomic_fetch_flip_fallback(atomic, bit, order);
}
return previous;
} }
template <typename Atomic> template <typename Atomic>
...@@ -332,77 +325,50 @@ bool atomic_fetch_flip_native( ...@@ -332,77 +325,50 @@ bool atomic_fetch_flip_native(
#else #else
template <typename Atomic> using atomic_fetch_set_native_fn = detail::atomic_fetch_set_fallback_fn;
bool atomic_fetch_set_native(Atomic&, std::size_t, std::memory_order) noexcept { FOLLY_INLINE_VARIABLE constexpr atomic_fetch_set_native_fn
// This should never be called on non x86_64 platforms. atomic_fetch_set_native{};
std::terminate();
}
template <typename Atomic>
bool atomic_fetch_reset_native(
Atomic&, std::size_t, std::memory_order) noexcept {
// This should never be called on non x86_64 platforms.
std::terminate();
}
template <typename Atomic>
bool atomic_fetch_flip_native(
Atomic&, std::size_t, std::memory_order) noexcept {
// This should never be called on non x86_64 platforms.
std::terminate();
}
#endif using atomic_fetch_reset_native_fn = detail::atomic_fetch_reset_fallback_fn;
FOLLY_INLINE_VARIABLE constexpr atomic_fetch_reset_native_fn
atomic_fetch_reset_native{};
} // namespace detail using atomic_fetch_flip_native_fn = detail::atomic_fetch_flip_fallback_fn;
FOLLY_INLINE_VARIABLE constexpr atomic_fetch_flip_native_fn
atomic_fetch_flip_native{};
#endif
template <typename Atomic> template <typename Atomic>
bool atomic_fetch_set(Atomic& atomic, std::size_t bit, std::memory_order mo) { void atomic_fetch_bit_op_check_(Atomic& atomic, std::size_t bit) {
using Integer = decltype(atomic.load()); using Integer = decltype(atomic.load());
static_assert(std::is_unsigned<Integer>{}, ""); static_assert(std::is_unsigned<Integer>{}, "");
static_assert(!std::is_const<Atomic>{}, ""); static_assert(!std::is_const<Atomic>{}, "");
assert(bit < (sizeof(Integer) * 8)); assert(bit < (sizeof(Integer) * 8));
(void)bit;
// do the optimized thing on x86 builds. Also, some versions of TSAN do not
// properly instrument the inline assembly, so avoid it when TSAN is enabled
if (folly::kIsArchAmd64 && !folly::kIsSanitizeThread) {
return detail::atomic_fetch_set_native(atomic, bit, mo);
} else {
// otherwise default to the default implementation using fetch_or()
return detail::atomic_fetch_set_fallback(atomic, bit, mo);
}
} }
template <typename Atomic> } // namespace detail
bool atomic_fetch_reset(Atomic& atomic, std::size_t bit, std::memory_order mo) {
using Integer = decltype(atomic.load());
static_assert(std::is_unsigned<Integer>{}, "");
static_assert(!std::is_const<Atomic>{}, "");
assert(bit < (sizeof(Integer) * 8));
// do the optimized thing on x86 builds. Also, some versions of TSAN do not template <typename Atomic>
// properly instrument the inline assembly, so avoid it when TSAN is enabled bool atomic_fetch_set_fn::operator()(
if (folly::kIsArchAmd64 && !folly::kIsSanitizeThread) { Atomic& atomic, std::size_t bit, std::memory_order mo) const {
return detail::atomic_fetch_reset_native(atomic, bit, mo); detail::atomic_fetch_bit_op_check_(atomic, bit);
} else { return detail::atomic_fetch_set_native(atomic, bit, mo);
// otherwise default to the default implementation using fetch_and()
return detail::atomic_fetch_reset_fallback(atomic, bit, mo);
}
} }
template <typename Atomic> template <typename Atomic>
bool atomic_fetch_flip(Atomic& atomic, std::size_t bit, std::memory_order mo) { bool atomic_fetch_reset_fn::operator()(
using Integer = decltype(atomic.load()); Atomic& atomic, std::size_t bit, std::memory_order mo) const {
static_assert(std::is_unsigned<Integer>{}, ""); detail::atomic_fetch_bit_op_check_(atomic, bit);
static_assert(!std::is_const<Atomic>{}, ""); return detail::atomic_fetch_reset_native(atomic, bit, mo);
assert(bit < (sizeof(Integer) * 8)); }
// do the optimized thing on x86 builds. Also, some versions of TSAN do not template <typename Atomic>
// properly instrument the inline assembly, so avoid it when TSAN is enabled bool atomic_fetch_flip_fn::operator()(
if (folly::kIsArchAmd64 && !folly::kIsSanitizeThread) { Atomic& atomic, std::size_t bit, std::memory_order mo) const {
return detail::atomic_fetch_flip_native(atomic, bit, mo); detail::atomic_fetch_bit_op_check_(atomic, bit);
} else { return detail::atomic_fetch_flip_native(atomic, bit, mo);
// otherwise default to the default implementation using fetch_and()
return detail::atomic_fetch_flip_fallback(atomic, bit, mo);
}
} }
} // namespace folly } // namespace folly
folly/synchronization/AtomicUtil.h
View file @ d60894bf
...@@ -19,6 +19,7 @@ ...@@ -19,6 +19,7 @@
#include <atomic> #include <atomic>
#include <cstdint> #include <cstdint>
#include <folly/Portability.h>
#include <folly/Traits.h> #include <folly/Traits.h>
namespace folly { namespace folly {
...@@ -64,11 +65,14 @@ bool atomic_compare_exchange_strong_explicit( ...@@ -64,11 +65,14 @@ bool atomic_compare_exchange_strong_explicit(
// Uses an optimized implementation when available, otherwise falling back to // Uses an optimized implementation when available, otherwise falling back to
// Atomic::fetch_or with mask. The optimization is currently available for // Atomic::fetch_or with mask. The optimization is currently available for
// std::atomic on x86, using the bts instruction. // std::atomic on x86, using the bts instruction.
template <typename Atomic> struct atomic_fetch_set_fn {
bool atomic_fetch_set( template <typename Atomic>
Atomic& atomic, bool operator()(
std::size_t bit, Atomic& atomic,
std::memory_order order = std::memory_order_seq_cst); std::size_t bit,
std::memory_order order = std::memory_order_seq_cst) const;
};
FOLLY_INLINE_VARIABLE constexpr atomic_fetch_set_fn atomic_fetch_set{};
// atomic_fetch_reset // atomic_fetch_reset
// //
...@@ -83,11 +87,14 @@ bool atomic_fetch_set( ...@@ -83,11 +87,14 @@ bool atomic_fetch_set(
// Uses an optimized implementation when available, otherwise falling back to // Uses an optimized implementation when available, otherwise falling back to
// Atomic::fetch_and with mask. The optimization is currently available for // Atomic::fetch_and with mask. The optimization is currently available for
// std::atomic on x86, using the btr instruction. // std::atomic on x86, using the btr instruction.
template <typename Atomic> struct atomic_fetch_reset_fn {
bool atomic_fetch_reset( template <typename Atomic>
Atomic& atomic, bool operator()(
std::size_t bit, Atomic& atomic,
std::memory_order order = std::memory_order_seq_cst); std::size_t bit,
std::memory_order order = std::memory_order_seq_cst) const;
};
FOLLY_INLINE_VARIABLE constexpr atomic_fetch_reset_fn atomic_fetch_reset{};
// atomic_fetch_flip // atomic_fetch_flip
// //
...@@ -101,11 +108,14 @@ bool atomic_fetch_reset( ...@@ -101,11 +108,14 @@ bool atomic_fetch_reset(
// Uses an optimized implementation when available, otherwise falling back to // Uses an optimized implementation when available, otherwise falling back to
// Atomic::fetch_xor with mask. The optimization is currently available for // Atomic::fetch_xor with mask. The optimization is currently available for
// std::atomic on x86, using the btc instruction. // std::atomic on x86, using the btc instruction.
template <typename Atomic> struct atomic_fetch_flip_fn {
bool atomic_fetch_flip( template <typename Atomic>
Atomic& atomic, bool operator()(
std::size_t bit, Atomic& atomic,
std::memory_order order = std::memory_order_seq_cst); std::size_t bit,
std::memory_order order = std::memory_order_seq_cst) const;
};
FOLLY_INLINE_VARIABLE constexpr atomic_fetch_flip_fn atomic_fetch_flip{};
} // namespace folly } // namespace folly
......
folly/synchronization/test/AtomicUtilBench.cpp
View file @ d60894bf
...@@ -119,15 +119,6 @@ FOLLY_ATOMIC_FETCH_BIT_OP_CHECK_FIX(flip, 16, 11) ...@@ -119,15 +119,6 @@ FOLLY_ATOMIC_FETCH_BIT_OP_CHECK_FIX(flip, 16, 11)
#undef FOLLY_ATOMIC_FETCH_BIT_OP_CHECK_FIX #undef FOLLY_ATOMIC_FETCH_BIT_OP_CHECK_FIX
#undef FOLLY_ATOMIC_FETCH_BIT_OP_CHECK_VAR #undef FOLLY_ATOMIC_FETCH_BIT_OP_CHECK_VAR
namespace atomic_util_access {
FOLLY_CREATE_FREE_INVOKER_SUITE(atomic_fetch_set, folly);
FOLLY_CREATE_FREE_INVOKER_SUITE(atomic_fetch_reset, folly);
FOLLY_CREATE_FREE_INVOKER_SUITE(atomic_fetch_flip, folly);
FOLLY_CREATE_FREE_INVOKER_SUITE(atomic_fetch_set_fallback, folly::detail);
FOLLY_CREATE_FREE_INVOKER_SUITE(atomic_fetch_reset_fallback, folly::detail);
FOLLY_CREATE_FREE_INVOKER_SUITE(atomic_fetch_flip_fallback, folly::detail);
} // namespace atomic_util_access
namespace { namespace {
enum class what { drop, keep, cond }; enum class what { drop, keep, cond };
...@@ -247,12 +238,12 @@ BENCHMARK(atomic_fetch_set_u8_var_drop_monopoly, iters) { ...@@ -247,12 +238,12 @@ BENCHMARK(atomic_fetch_set_u8_var_drop_monopoly, iters) {
} }
BENCHMARK(atomic_fetch_set_u8_var_drop_fallback, iters) { BENCHMARK(atomic_fetch_set_u8_var_drop_fallback, iters) {
auto op = atomic_util_access::atomic_fetch_set_fallback; auto op = folly::detail::atomic_fetch_set_fallback;
atomic_fetch_op_var_(uint8_t(0), op, what_constant<what::drop>{}, iters); atomic_fetch_op_var_(uint8_t(0), op, what_constant<what::drop>{}, iters);
} }
BENCHMARK(atomic_fetch_set_u8_var_drop_native, iters) { BENCHMARK(atomic_fetch_set_u8_var_drop_native, iters) {
auto op = atomic_util_access::atomic_fetch_set; auto op = folly::atomic_fetch_set;
atomic_fetch_op_var_(uint8_t(0), op, what_constant<what::drop>{}, iters); atomic_fetch_op_var_(uint8_t(0), op, what_constant<what::drop>{}, iters);
} }
...@@ -262,12 +253,12 @@ BENCHMARK(atomic_fetch_set_u8_var_keep_monopoly, iters) { ...@@ -262,12 +253,12 @@ BENCHMARK(atomic_fetch_set_u8_var_keep_monopoly, iters) {
} }
BENCHMARK(atomic_fetch_set_u8_var_keep_fallback, iters) { BENCHMARK(atomic_fetch_set_u8_var_keep_fallback, iters) {
auto op = atomic_util_access::atomic_fetch_set_fallback; auto op = folly::detail::atomic_fetch_set_fallback;
atomic_fetch_op_var_(uint8_t(0), op, what_constant<what::keep>{}, iters); atomic_fetch_op_var_(uint8_t(0), op, what_constant<what::keep>{}, iters);
} }
BENCHMARK(atomic_fetch_set_u8_var_keep_native, iters) { BENCHMARK(atomic_fetch_set_u8_var_keep_native, iters) {
auto op = atomic_util_access::atomic_fetch_set; auto op = folly::atomic_fetch_set;
atomic_fetch_op_var_(uint8_t(0), op, what_constant<what::keep>{}, iters); atomic_fetch_op_var_(uint8_t(0), op, what_constant<what::keep>{}, iters);
} }
...@@ -277,12 +268,12 @@ BENCHMARK(atomic_fetch_set_u8_var_cond_monopoly, iters) { ...@@ -277,12 +268,12 @@ BENCHMARK(atomic_fetch_set_u8_var_cond_monopoly, iters) {
} }
BENCHMARK(atomic_fetch_set_u8_var_cond_fallback, iters) { BENCHMARK(atomic_fetch_set_u8_var_cond_fallback, iters) {
auto op = atomic_util_access::atomic_fetch_set_fallback; auto op = folly::detail::atomic_fetch_set_fallback;
atomic_fetch_op_var_(uint8_t(0), op, what_constant<what::cond>{}, iters); atomic_fetch_op_var_(uint8_t(0), op, what_constant<what::cond>{}, iters);
} }
BENCHMARK(atomic_fetch_set_u8_var_cond_native, iters) { BENCHMARK(atomic_fetch_set_u8_var_cond_native, iters) {
auto op = atomic_util_access::atomic_fetch_set; auto op = folly::atomic_fetch_set;
atomic_fetch_op_var_(uint8_t(0), op, what_constant<what::cond>{}, iters); atomic_fetch_op_var_(uint8_t(0), op, what_constant<what::cond>{}, iters);
} }
...@@ -294,12 +285,12 @@ BENCHMARK(atomic_fetch_set_u8_fix_drop_monopoly, iters) { ...@@ -294,12 +285,12 @@ BENCHMARK(atomic_fetch_set_u8_fix_drop_monopoly, iters) {
} }
BENCHMARK(atomic_fetch_set_u8_fix_drop_fallback, iters) { BENCHMARK(atomic_fetch_set_u8_fix_drop_fallback, iters) {
auto op = atomic_util_access::atomic_fetch_set_fallback; auto op = folly::detail::atomic_fetch_set_fallback;
atomic_fetch_op_fix_(uint8_t(0), op, what_constant<what::drop>{}, iters); atomic_fetch_op_fix_(uint8_t(0), op, what_constant<what::drop>{}, iters);
} }
BENCHMARK(atomic_fetch_set_u8_fix_drop_native, iters) { BENCHMARK(atomic_fetch_set_u8_fix_drop_native, iters) {
auto op = atomic_util_access::atomic_fetch_set; auto op = folly::atomic_fetch_set;
atomic_fetch_op_fix_(uint8_t(0), op, what_constant<what::drop>{}, iters); atomic_fetch_op_fix_(uint8_t(0), op, what_constant<what::drop>{}, iters);
} }
...@@ -309,12 +300,12 @@ BENCHMARK(atomic_fetch_set_u8_fix_keep_monopoly, iters) { ...@@ -309,12 +300,12 @@ BENCHMARK(atomic_fetch_set_u8_fix_keep_monopoly, iters) {
} }
BENCHMARK(atomic_fetch_set_u8_fix_keep_fallback, iters) { BENCHMARK(atomic_fetch_set_u8_fix_keep_fallback, iters) {
auto op = atomic_util_access::atomic_fetch_set_fallback; auto op = folly::detail::atomic_fetch_set_fallback;
atomic_fetch_op_fix_(uint8_t(0), op, what_constant<what::keep>{}, iters); atomic_fetch_op_fix_(uint8_t(0), op, what_constant<what::keep>{}, iters);
} }
BENCHMARK(atomic_fetch_set_u8_fix_keep_native, iters) { BENCHMARK(atomic_fetch_set_u8_fix_keep_native, iters) {
auto op = atomic_util_access::atomic_fetch_set; auto op = folly::atomic_fetch_set;
atomic_fetch_op_fix_(uint8_t(0), op, what_constant<what::keep>{}, iters); atomic_fetch_op_fix_(uint8_t(0), op, what_constant<what::keep>{}, iters);
} }
...@@ -324,12 +315,12 @@ BENCHMARK(atomic_fetch_set_u8_fix_cond_monopoly, iters) { ...@@ -324,12 +315,12 @@ BENCHMARK(atomic_fetch_set_u8_fix_cond_monopoly, iters) {
} }
BENCHMARK(atomic_fetch_set_u8_fix_cond_fallback, iters) { BENCHMARK(atomic_fetch_set_u8_fix_cond_fallback, iters) {
auto op = atomic_util_access::atomic_fetch_set_fallback; auto op = folly::detail::atomic_fetch_set_fallback;
atomic_fetch_op_fix_(uint8_t(0), op, what_constant<what::cond>{}, iters); atomic_fetch_op_fix_(uint8_t(0), op, what_constant<what::cond>{}, iters);
} }
BENCHMARK(atomic_fetch_set_u8_fix_cond_native, iters) { BENCHMARK(atomic_fetch_set_u8_fix_cond_native, iters) {
auto op = atomic_util_access::atomic_fetch_set; auto op = folly::atomic_fetch_set;
atomic_fetch_op_fix_(uint8_t(0), op, what_constant<what::cond>{}, iters); atomic_fetch_op_fix_(uint8_t(0), op, what_constant<what::cond>{}, iters);
} }
...@@ -341,12 +332,12 @@ BENCHMARK(atomic_fetch_set_u16_var_drop_monopoly, iters) { ...@@ -341,12 +332,12 @@ BENCHMARK(atomic_fetch_set_u16_var_drop_monopoly, iters) {
} }
BENCHMARK(atomic_fetch_set_u16_var_drop_fallback, iters) { BENCHMARK(atomic_fetch_set_u16_var_drop_fallback, iters) {
auto op = atomic_util_access::atomic_fetch_set_fallback; auto op = folly::detail::atomic_fetch_set_fallback;
atomic_fetch_op_var_(uint16_t(0), op, what_constant<what::drop>{}, iters); atomic_fetch_op_var_(uint16_t(0), op, what_constant<what::drop>{}, iters);
} }
BENCHMARK(atomic_fetch_set_u16_var_drop_native, iters) { BENCHMARK(atomic_fetch_set_u16_var_drop_native, iters) {
auto op = atomic_util_access::atomic_fetch_set; auto op = folly::atomic_fetch_set;
atomic_fetch_op_var_(uint16_t(0), op, what_constant<what::drop>{}, iters); atomic_fetch_op_var_(uint16_t(0), op, what_constant<what::drop>{}, iters);
} }
...@@ -356,12 +347,12 @@ BENCHMARK(atomic_fetch_set_u16_var_keep_monopoly, iters) { ...@@ -356,12 +347,12 @@ BENCHMARK(atomic_fetch_set_u16_var_keep_monopoly, iters) {
} }
BENCHMARK(atomic_fetch_set_u16_var_keep_fallback, iters) { BENCHMARK(atomic_fetch_set_u16_var_keep_fallback, iters) {
auto op = atomic_util_access::atomic_fetch_set_fallback; auto op = folly::detail::atomic_fetch_set_fallback;
atomic_fetch_op_var_(uint16_t(0), op, what_constant<what::keep>{}, iters); atomic_fetch_op_var_(uint16_t(0), op, what_constant<what::keep>{}, iters);
} }
BENCHMARK(atomic_fetch_set_u16_var_keep_native, iters) { BENCHMARK(atomic_fetch_set_u16_var_keep_native, iters) {
auto op = atomic_util_access::atomic_fetch_set; auto op = folly::atomic_fetch_set;
atomic_fetch_op_var_(uint16_t(0), op, what_constant<what::keep>{}, iters); atomic_fetch_op_var_(uint16_t(0), op, what_constant<what::keep>{}, iters);
} }
...@@ -371,12 +362,12 @@ BENCHMARK(atomic_fetch_set_u16_var_cond_monopoly, iters) { ...@@ -371,12 +362,12 @@ BENCHMARK(atomic_fetch_set_u16_var_cond_monopoly, iters) {
} }
BENCHMARK(atomic_fetch_set_u16_var_cond_fallback, iters) { BENCHMARK(atomic_fetch_set_u16_var_cond_fallback, iters) {
auto op = atomic_util_access::atomic_fetch_set_fallback; auto op = folly::detail::atomic_fetch_set_fallback;
atomic_fetch_op_var_(uint16_t(0), op, what_constant<what::cond>{}, iters); atomic_fetch_op_var_(uint16_t(0), op, what_constant<what::cond>{}, iters);
} }
BENCHMARK(atomic_fetch_set_u16_var_cond_native, iters) { BENCHMARK(atomic_fetch_set_u16_var_cond_native, iters) {
auto op = atomic_util_access::atomic_fetch_set; auto op = folly::atomic_fetch_set;
atomic_fetch_op_var_(uint16_t(0), op, what_constant<what::cond>{}, iters); atomic_fetch_op_var_(uint16_t(0), op, what_constant<what::cond>{}, iters);
} }
...@@ -388,12 +379,12 @@ BENCHMARK(atomic_fetch_set_u16_fix_drop_monopoly, iters) { ...@@ -388,12 +379,12 @@ BENCHMARK(atomic_fetch_set_u16_fix_drop_monopoly, iters) {
} }
BENCHMARK(atomic_fetch_set_u16_fix_drop_fallback, iters) { BENCHMARK(atomic_fetch_set_u16_fix_drop_fallback, iters) {
auto op = atomic_util_access::atomic_fetch_set_fallback; auto op = folly::detail::atomic_fetch_set_fallback;
atomic_fetch_op_fix_(uint16_t(0), op, what_constant<what::drop>{}, iters); atomic_fetch_op_fix_(uint16_t(0), op, what_constant<what::drop>{}, iters);
} }
BENCHMARK(atomic_fetch_set_u16_fix_drop_native, iters) { BENCHMARK(atomic_fetch_set_u16_fix_drop_native, iters) {
auto op = atomic_util_access::atomic_fetch_set; auto op = folly::atomic_fetch_set;
atomic_fetch_op_fix_(uint16_t(0), op, what_constant<what::drop>{}, iters); atomic_fetch_op_fix_(uint16_t(0), op, what_constant<what::drop>{}, iters);
} }
...@@ -403,12 +394,12 @@ BENCHMARK(atomic_fetch_set_u16_fix_keep_monopoly, iters) { ...@@ -403,12 +394,12 @@ BENCHMARK(atomic_fetch_set_u16_fix_keep_monopoly, iters) {
} }
BENCHMARK(atomic_fetch_set_u16_fix_keep_fallback, iters) { BENCHMARK(atomic_fetch_set_u16_fix_keep_fallback, iters) {
auto op = atomic_util_access::atomic_fetch_set_fallback; auto op = folly::detail::atomic_fetch_set_fallback;
atomic_fetch_op_fix_(uint16_t(0), op, what_constant<what::keep>{}, iters); atomic_fetch_op_fix_(uint16_t(0), op, what_constant<what::keep>{}, iters);
} }
BENCHMARK(atomic_fetch_set_u16_fix_keep_native, iters) { BENCHMARK(atomic_fetch_set_u16_fix_keep_native, iters) {
auto op = atomic_util_access::atomic_fetch_set; auto op = folly::atomic_fetch_set;
atomic_fetch_op_fix_(uint16_t(0), op, what_constant<what::keep>{}, iters); atomic_fetch_op_fix_(uint16_t(0), op, what_constant<what::keep>{}, iters);
} }
...@@ -418,12 +409,12 @@ BENCHMARK(atomic_fetch_set_u16_fix_cond_monopoly, iters) { ...@@ -418,12 +409,12 @@ BENCHMARK(atomic_fetch_set_u16_fix_cond_monopoly, iters) {
} }
BENCHMARK(atomic_fetch_set_u16_fix_cond_fallback, iters) { BENCHMARK(atomic_fetch_set_u16_fix_cond_fallback, iters) {
auto op = atomic_util_access::atomic_fetch_set_fallback; auto op = folly::detail::atomic_fetch_set_fallback;
atomic_fetch_op_fix_(uint16_t(0), op, what_constant<what::cond>{}, iters); atomic_fetch_op_fix_(uint16_t(0), op, what_constant<what::cond>{}, iters);
} }
BENCHMARK(atomic_fetch_set_u16_fix_cond_native, iters) { BENCHMARK(atomic_fetch_set_u16_fix_cond_native, iters) {
auto op = atomic_util_access::atomic_fetch_set; auto op = folly::atomic_fetch_set;
atomic_fetch_op_fix_(uint16_t(0), op, what_constant<what::cond>{}, iters); atomic_fetch_op_fix_(uint16_t(0), op, what_constant<what::cond>{}, iters);
} }
......
folly/synchronization/test/AtomicUtilTest.cpp
View file @ d60894bf
...@@ -105,15 +105,6 @@ TEST_F(AtomicCompareExchangeSuccTest, examples) { ...@@ -105,15 +105,6 @@ TEST_F(AtomicCompareExchangeSuccTest, examples) {
} }
} }
namespace access {
FOLLY_CREATE_FREE_INVOKER_SUITE(atomic_fetch_set, folly);
FOLLY_CREATE_FREE_INVOKER_SUITE(atomic_fetch_reset, folly);
FOLLY_CREATE_FREE_INVOKER_SUITE(atomic_fetch_flip, folly);
FOLLY_CREATE_FREE_INVOKER_SUITE(atomic_fetch_set_fallback, folly::detail);
FOLLY_CREATE_FREE_INVOKER_SUITE(atomic_fetch_reset_fallback, folly::detail);
FOLLY_CREATE_FREE_INVOKER_SUITE(atomic_fetch_flip_fallback, folly::detail);
} // namespace access
namespace { namespace {
template <typename I> template <typename I>
...@@ -125,7 +116,7 @@ struct with_seq_cst : private I { ...@@ -125,7 +116,7 @@ struct with_seq_cst : private I {
} }
}; };
template <typename Integer, typename Op = access::atomic_fetch_set_fn> template <typename Integer, typename Op = folly::atomic_fetch_set_fn>
void atomic_fetch_set_basic(Op fetch_set = {}) { void atomic_fetch_set_basic(Op fetch_set = {}) {
{ {
auto&& atomic = std::atomic<Integer>{0}; auto&& atomic = std::atomic<Integer>{0};
...@@ -165,7 +156,7 @@ void atomic_fetch_set_basic(Op fetch_set = {}) { ...@@ -165,7 +156,7 @@ void atomic_fetch_set_basic(Op fetch_set = {}) {
} }
} }
template <typename Integer, typename Op = access::atomic_fetch_reset_fn> template <typename Integer, typename Op = folly::atomic_fetch_reset_fn>
void atomic_fetch_reset_basic(Op fetch_reset = {}) { void atomic_fetch_reset_basic(Op fetch_reset = {}) {
{ {
auto&& atomic = std::atomic<Integer>{0}; auto&& atomic = std::atomic<Integer>{0};
...@@ -188,7 +179,7 @@ void atomic_fetch_reset_basic(Op fetch_reset = {}) { ...@@ -188,7 +179,7 @@ void atomic_fetch_reset_basic(Op fetch_reset = {}) {
} }
} }
template <typename Integer, typename Op = access::atomic_fetch_flip_fn> template <typename Integer, typename Op = folly::atomic_fetch_flip_fn>
void atomic_fetch_flip_basic(Op fetch_flip = {}) { void atomic_fetch_flip_basic(Op fetch_flip = {}) {
{ {
auto&& atomic = std::atomic<Integer>{0}; auto&& atomic = std::atomic<Integer>{0};
...@@ -214,6 +205,8 @@ void atomic_fetch_flip_basic(Op fetch_flip = {}) { ...@@ -214,6 +205,8 @@ void atomic_fetch_flip_basic(Op fetch_flip = {}) {
template <typename Integer> template <typename Integer>
class Atomic { class Atomic {
public: public:
using value_type = Integer;
Integer fetch_or( Integer fetch_or(
Integer value, std::memory_order = std::memory_order_seq_cst) { Integer value, std::memory_order = std::memory_order_seq_cst) {
++counts.set; ++counts.set;
...@@ -244,7 +237,7 @@ class Atomic { ...@@ -244,7 +237,7 @@ class Atomic {
counts_ counts; counts_ counts;
}; };
template <typename Integer, typename Op = access::atomic_fetch_set_fn> template <typename Integer, typename Op = folly::atomic_fetch_set_fn>
void atomic_fetch_set_non_std_atomic(Op fetch_set = {}) { void atomic_fetch_set_non_std_atomic(Op fetch_set = {}) {
auto atomic = Atomic<Integer>{}; auto atomic = Atomic<Integer>{};
auto& sets = atomic.counts.set; auto& sets = atomic.counts.set;
...@@ -264,7 +257,7 @@ void atomic_fetch_set_non_std_atomic(Op fetch_set = {}) { ...@@ -264,7 +257,7 @@ void atomic_fetch_set_non_std_atomic(Op fetch_set = {}) {
EXPECT_EQ(atomic.integer_, 0b101); EXPECT_EQ(atomic.integer_, 0b101);
} }
template <typename Integer, typename Op = access::atomic_fetch_reset_fn> template <typename Integer, typename Op = folly::atomic_fetch_reset_fn>
void atomic_fetch_reset_non_std_atomic(Op fetch_reset = {}) { void atomic_fetch_reset_non_std_atomic(Op fetch_reset = {}) {
auto atomic = Atomic<Integer>{}; auto atomic = Atomic<Integer>{};
auto& sets = atomic.counts.set; auto& sets = atomic.counts.set;
...@@ -285,7 +278,7 @@ void atomic_fetch_reset_non_std_atomic(Op fetch_reset = {}) { ...@@ -285,7 +278,7 @@ void atomic_fetch_reset_non_std_atomic(Op fetch_reset = {}) {
EXPECT_EQ(atomic.integer_, 0b010); EXPECT_EQ(atomic.integer_, 0b010);
} }
template <typename Integer, typename Op = access::atomic_fetch_flip_fn> template <typename Integer, typename Op = folly::atomic_fetch_flip_fn>
void atomic_fetch_flip_non_std_atomic(Op fetch_flip = {}) { void atomic_fetch_flip_non_std_atomic(Op fetch_flip = {}) {
auto atomic = Atomic<Integer>{}; auto atomic = Atomic<Integer>{};
auto& sets = atomic.counts.set; auto& sets = atomic.counts.set;
...@@ -354,7 +347,7 @@ TEST_F(AtomicFetchFlipTest, EnsureFetchXorUsed) { ...@@ -354,7 +347,7 @@ TEST_F(AtomicFetchFlipTest, EnsureFetchXorUsed) {
} }
TEST_F(AtomicFetchSetTest, FetchSetFallback) { TEST_F(AtomicFetchSetTest, FetchSetFallback) {
auto fetch_set = with_seq_cst{access::atomic_fetch_set_fallback}; auto fetch_set = with_seq_cst{folly::detail::atomic_fetch_set_fallback};
atomic_fetch_set_basic<std::uint16_t>(fetch_set); atomic_fetch_set_basic<std::uint16_t>(fetch_set);
atomic_fetch_set_basic<std::uint32_t>(fetch_set); atomic_fetch_set_basic<std::uint32_t>(fetch_set);
...@@ -368,7 +361,7 @@ TEST_F(AtomicFetchSetTest, FetchSetFallback) { ...@@ -368,7 +361,7 @@ TEST_F(AtomicFetchSetTest, FetchSetFallback) {
} }
TEST_F(AtomicFetchResetTest, FetchResetFallback) { TEST_F(AtomicFetchResetTest, FetchResetFallback) {
auto fetch_reset = with_seq_cst{access::atomic_fetch_reset_fallback}; auto fetch_reset = with_seq_cst{folly::detail::atomic_fetch_reset_fallback};
atomic_fetch_reset_basic<std::uint16_t>(fetch_reset); atomic_fetch_reset_basic<std::uint16_t>(fetch_reset);
atomic_fetch_reset_basic<std::uint32_t>(fetch_reset); atomic_fetch_reset_basic<std::uint32_t>(fetch_reset);
...@@ -382,7 +375,7 @@ TEST_F(AtomicFetchResetTest, FetchResetFallback) { ...@@ -382,7 +375,7 @@ TEST_F(AtomicFetchResetTest, FetchResetFallback) {
} }
TEST_F(AtomicFetchFlipTest, FetchFlipFallback) { TEST_F(AtomicFetchFlipTest, FetchFlipFallback) {
auto fetch_flip = with_seq_cst{access::atomic_fetch_flip_fallback}; auto fetch_flip = with_seq_cst{folly::detail::atomic_fetch_flip_fallback};
atomic_fetch_flip_basic<std::uint16_t>(fetch_flip); atomic_fetch_flip_basic<std::uint16_t>(fetch_flip);
atomic_fetch_flip_basic<std::uint32_t>(fetch_flip); atomic_fetch_flip_basic<std::uint32_t>(fetch_flip);
...@@ -396,7 +389,7 @@ TEST_F(AtomicFetchFlipTest, FetchFlipFallback) { ...@@ -396,7 +389,7 @@ TEST_F(AtomicFetchFlipTest, FetchFlipFallback) {
} }
TEST_F(AtomicFetchSetTest, FetchSetDefault) { TEST_F(AtomicFetchSetTest, FetchSetDefault) {
auto fetch_set = access::atomic_fetch_set; auto fetch_set = folly::atomic_fetch_set;
atomic_fetch_set_basic<std::uint16_t>(fetch_set); atomic_fetch_set_basic<std::uint16_t>(fetch_set);
atomic_fetch_set_basic<std::uint32_t>(fetch_set); atomic_fetch_set_basic<std::uint32_t>(fetch_set);
...@@ -410,7 +403,7 @@ TEST_F(AtomicFetchSetTest, FetchSetDefault) { ...@@ -410,7 +403,7 @@ TEST_F(AtomicFetchSetTest, FetchSetDefault) {
} }
TEST_F(AtomicFetchResetTest, FetchResetDefault) { TEST_F(AtomicFetchResetTest, FetchResetDefault) {
auto fetch_reset = access::atomic_fetch_reset; auto fetch_reset = folly::atomic_fetch_reset;
atomic_fetch_reset_basic<std::uint16_t>(fetch_reset); atomic_fetch_reset_basic<std::uint16_t>(fetch_reset);
atomic_fetch_reset_basic<std::uint32_t>(fetch_reset); atomic_fetch_reset_basic<std::uint32_t>(fetch_reset);
...@@ -424,7 +417,7 @@ TEST_F(AtomicFetchResetTest, FetchResetDefault) { ...@@ -424,7 +417,7 @@ TEST_F(AtomicFetchResetTest, FetchResetDefault) {
} }
TEST_F(AtomicFetchFlipTest, FetchFlipDefault) { TEST_F(AtomicFetchFlipTest, FetchFlipDefault) {
auto fetch_flip = access::atomic_fetch_flip; auto fetch_flip = folly::atomic_fetch_flip;
atomic_fetch_flip_basic<std::uint16_t>(fetch_flip); atomic_fetch_flip_basic<std::uint16_t>(fetch_flip);
atomic_fetch_flip_basic<std::uint32_t>(fetch_flip); atomic_fetch_flip_basic<std::uint32_t>(fetch_flip);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment