Skip to content

F
folly
Commit d60894bf authored by Yedidya Feldblum's avatar Yedidya Feldblum Committed by Facebook GitHub Bot
Browse files
Options

condense the atomic-fetch-bit-op implementations 

Summary: Coalesce the inline-asm. Use invokers.

Reviewed By: Gownta

Differential Revision: D32302511

fbshipit-source-id: 6edf75c6ef820ed80e751d965174e5acfaac957e
parent 80c05933
Expand all Show whitespace changes
Inline Side-by-side
Showing with 188 additions and 228 deletions
folly/synchronization/AtomicUtil-inl.h
View file @ d60894bf
This diff is collapsed.
folly/synchronization/AtomicUtil.h
View file @ d60894bf
......@@ -19,6 +19,7 @@
#include <atomic>
#include <cstdint>
#include <folly/Portability.h>
#include <folly/Traits.h>
namespace folly {
......@@ -64,11 +65,14 @@ bool atomic_compare_exchange_strong_explicit(
// Uses an optimized implementation when available, otherwise falling back to
// Atomic::fetch_or with mask. The optimization is currently available for
// std::atomic on x86, using the bts instruction.
template <typename Atomic>
bool atomic_fetch_set(
struct atomic_fetch_set_fn {
template <typename Atomic>
bool operator()(
Atomic& atomic,
std::size_t bit,
std::memory_order order = std::memory_order_seq_cst);
std::memory_order order = std::memory_order_seq_cst) const;
};
FOLLY_INLINE_VARIABLE constexpr atomic_fetch_set_fn atomic_fetch_set{};
// atomic_fetch_reset
//
......@@ -83,11 +87,14 @@ bool atomic_fetch_set(
// Uses an optimized implementation when available, otherwise falling back to
// Atomic::fetch_and with mask. The optimization is currently available for
// std::atomic on x86, using the btr instruction.
template <typename Atomic>
bool atomic_fetch_reset(
struct atomic_fetch_reset_fn {
template <typename Atomic>
bool operator()(
Atomic& atomic,
std::size_t bit,
std::memory_order order = std::memory_order_seq_cst);
std::memory_order order = std::memory_order_seq_cst) const;
};
FOLLY_INLINE_VARIABLE constexpr atomic_fetch_reset_fn atomic_fetch_reset{};
// atomic_fetch_flip
//
......@@ -101,11 +108,14 @@ bool atomic_fetch_reset(
// Uses an optimized implementation when available, otherwise falling back to
// Atomic::fetch_xor with mask. The optimization is currently available for
// std::atomic on x86, using the btc instruction.
template <typename Atomic>
bool atomic_fetch_flip(
struct atomic_fetch_flip_fn {
template <typename Atomic>
bool operator()(
Atomic& atomic,
std::size_t bit,
std::memory_order order = std::memory_order_seq_cst);
std::memory_order order = std::memory_order_seq_cst) const;
};
FOLLY_INLINE_VARIABLE constexpr atomic_fetch_flip_fn atomic_fetch_flip{};
} // namespace folly
......
folly/synchronization/test/AtomicUtilBench.cpp
View file @ d60894bf
......@@ -119,15 +119,6 @@ FOLLY_ATOMIC_FETCH_BIT_OP_CHECK_FIX(flip, 16, 11)
#undef FOLLY_ATOMIC_FETCH_BIT_OP_CHECK_FIX
#undef FOLLY_ATOMIC_FETCH_BIT_OP_CHECK_VAR
namespace atomic_util_access {
FOLLY_CREATE_FREE_INVOKER_SUITE(atomic_fetch_set, folly);
FOLLY_CREATE_FREE_INVOKER_SUITE(atomic_fetch_reset, folly);
FOLLY_CREATE_FREE_INVOKER_SUITE(atomic_fetch_flip, folly);
FOLLY_CREATE_FREE_INVOKER_SUITE(atomic_fetch_set_fallback, folly::detail);
FOLLY_CREATE_FREE_INVOKER_SUITE(atomic_fetch_reset_fallback, folly::detail);
FOLLY_CREATE_FREE_INVOKER_SUITE(atomic_fetch_flip_fallback, folly::detail);
} // namespace atomic_util_access
namespace {
enum class what { drop, keep, cond };
......@@ -247,12 +238,12 @@ BENCHMARK(atomic_fetch_set_u8_var_drop_monopoly, iters) {
}
BENCHMARK(atomic_fetch_set_u8_var_drop_fallback, iters) {
auto op = atomic_util_access::atomic_fetch_set_fallback;
auto op = folly::detail::atomic_fetch_set_fallback;
atomic_fetch_op_var_(uint8_t(0), op, what_constant<what::drop>{}, iters);
}
BENCHMARK(atomic_fetch_set_u8_var_drop_native, iters) {
auto op = atomic_util_access::atomic_fetch_set;
auto op = folly::atomic_fetch_set;
atomic_fetch_op_var_(uint8_t(0), op, what_constant<what::drop>{}, iters);
}
......@@ -262,12 +253,12 @@ BENCHMARK(atomic_fetch_set_u8_var_keep_monopoly, iters) {
}
BENCHMARK(atomic_fetch_set_u8_var_keep_fallback, iters) {
auto op = atomic_util_access::atomic_fetch_set_fallback;
auto op = folly::detail::atomic_fetch_set_fallback;
atomic_fetch_op_var_(uint8_t(0), op, what_constant<what::keep>{}, iters);
}
BENCHMARK(atomic_fetch_set_u8_var_keep_native, iters) {
auto op = atomic_util_access::atomic_fetch_set;
auto op = folly::atomic_fetch_set;
atomic_fetch_op_var_(uint8_t(0), op, what_constant<what::keep>{}, iters);
}
......@@ -277,12 +268,12 @@ BENCHMARK(atomic_fetch_set_u8_var_cond_monopoly, iters) {
}
BENCHMARK(atomic_fetch_set_u8_var_cond_fallback, iters) {
auto op = atomic_util_access::atomic_fetch_set_fallback;
auto op = folly::detail::atomic_fetch_set_fallback;
atomic_fetch_op_var_(uint8_t(0), op, what_constant<what::cond>{}, iters);
}
BENCHMARK(atomic_fetch_set_u8_var_cond_native, iters) {
auto op = atomic_util_access::atomic_fetch_set;
auto op = folly::atomic_fetch_set;
atomic_fetch_op_var_(uint8_t(0), op, what_constant<what::cond>{}, iters);
}
......@@ -294,12 +285,12 @@ BENCHMARK(atomic_fetch_set_u8_fix_drop_monopoly, iters) {
}
BENCHMARK(atomic_fetch_set_u8_fix_drop_fallback, iters) {
auto op = atomic_util_access::atomic_fetch_set_fallback;
auto op = folly::detail::atomic_fetch_set_fallback;
atomic_fetch_op_fix_(uint8_t(0), op, what_constant<what::drop>{}, iters);
}
BENCHMARK(atomic_fetch_set_u8_fix_drop_native, iters) {
auto op = atomic_util_access::atomic_fetch_set;
auto op = folly::atomic_fetch_set;
atomic_fetch_op_fix_(uint8_t(0), op, what_constant<what::drop>{}, iters);
}
......@@ -309,12 +300,12 @@ BENCHMARK(atomic_fetch_set_u8_fix_keep_monopoly, iters) {
}
BENCHMARK(atomic_fetch_set_u8_fix_keep_fallback, iters) {
auto op = atomic_util_access::atomic_fetch_set_fallback;
auto op = folly::detail::atomic_fetch_set_fallback;
atomic_fetch_op_fix_(uint8_t(0), op, what_constant<what::keep>{}, iters);
}
BENCHMARK(atomic_fetch_set_u8_fix_keep_native, iters) {
auto op = atomic_util_access::atomic_fetch_set;
auto op = folly::atomic_fetch_set;
atomic_fetch_op_fix_(uint8_t(0), op, what_constant<what::keep>{}, iters);
}
......@@ -324,12 +315,12 @@ BENCHMARK(atomic_fetch_set_u8_fix_cond_monopoly, iters) {
}
BENCHMARK(atomic_fetch_set_u8_fix_cond_fallback, iters) {
auto op = atomic_util_access::atomic_fetch_set_fallback;
auto op = folly::detail::atomic_fetch_set_fallback;
atomic_fetch_op_fix_(uint8_t(0), op, what_constant<what::cond>{}, iters);
}
BENCHMARK(atomic_fetch_set_u8_fix_cond_native, iters) {
auto op = atomic_util_access::atomic_fetch_set;
auto op = folly::atomic_fetch_set;
atomic_fetch_op_fix_(uint8_t(0), op, what_constant<what::cond>{}, iters);
}
......@@ -341,12 +332,12 @@ BENCHMARK(atomic_fetch_set_u16_var_drop_monopoly, iters) {
}
BENCHMARK(atomic_fetch_set_u16_var_drop_fallback, iters) {
auto op = atomic_util_access::atomic_fetch_set_fallback;
auto op = folly::detail::atomic_fetch_set_fallback;
atomic_fetch_op_var_(uint16_t(0), op, what_constant<what::drop>{}, iters);
}
BENCHMARK(atomic_fetch_set_u16_var_drop_native, iters) {
auto op = atomic_util_access::atomic_fetch_set;
auto op = folly::atomic_fetch_set;
atomic_fetch_op_var_(uint16_t(0), op, what_constant<what::drop>{}, iters);
}
......@@ -356,12 +347,12 @@ BENCHMARK(atomic_fetch_set_u16_var_keep_monopoly, iters) {
}
BENCHMARK(atomic_fetch_set_u16_var_keep_fallback, iters) {
auto op = atomic_util_access::atomic_fetch_set_fallback;
auto op = folly::detail::atomic_fetch_set_fallback;
atomic_fetch_op_var_(uint16_t(0), op, what_constant<what::keep>{}, iters);
}
BENCHMARK(atomic_fetch_set_u16_var_keep_native, iters) {
auto op = atomic_util_access::atomic_fetch_set;
auto op = folly::atomic_fetch_set;
atomic_fetch_op_var_(uint16_t(0), op, what_constant<what::keep>{}, iters);
}
......@@ -371,12 +362,12 @@ BENCHMARK(atomic_fetch_set_u16_var_cond_monopoly, iters) {
}
BENCHMARK(atomic_fetch_set_u16_var_cond_fallback, iters) {
auto op = atomic_util_access::atomic_fetch_set_fallback;
auto op = folly::detail::atomic_fetch_set_fallback;
atomic_fetch_op_var_(uint16_t(0), op, what_constant<what::cond>{}, iters);
}
BENCHMARK(atomic_fetch_set_u16_var_cond_native, iters) {
auto op = atomic_util_access::atomic_fetch_set;
auto op = folly::atomic_fetch_set;
atomic_fetch_op_var_(uint16_t(0), op, what_constant<what::cond>{}, iters);
}
......@@ -388,12 +379,12 @@ BENCHMARK(atomic_fetch_set_u16_fix_drop_monopoly, iters) {
}
BENCHMARK(atomic_fetch_set_u16_fix_drop_fallback, iters) {
auto op = atomic_util_access::atomic_fetch_set_fallback;
auto op = folly::detail::atomic_fetch_set_fallback;
atomic_fetch_op_fix_(uint16_t(0), op, what_constant<what::drop>{}, iters);
}
BENCHMARK(atomic_fetch_set_u16_fix_drop_native, iters) {
auto op = atomic_util_access::atomic_fetch_set;
auto op = folly::atomic_fetch_set;
atomic_fetch_op_fix_(uint16_t(0), op, what_constant<what::drop>{}, iters);
}
......@@ -403,12 +394,12 @@ BENCHMARK(atomic_fetch_set_u16_fix_keep_monopoly, iters) {
}
BENCHMARK(atomic_fetch_set_u16_fix_keep_fallback, iters) {
auto op = atomic_util_access::atomic_fetch_set_fallback;
auto op = folly::detail::atomic_fetch_set_fallback;
atomic_fetch_op_fix_(uint16_t(0), op, what_constant<what::keep>{}, iters);
}
BENCHMARK(atomic_fetch_set_u16_fix_keep_native, iters) {
auto op = atomic_util_access::atomic_fetch_set;
auto op = folly::atomic_fetch_set;
atomic_fetch_op_fix_(uint16_t(0), op, what_constant<what::keep>{}, iters);
}
......@@ -418,12 +409,12 @@ BENCHMARK(atomic_fetch_set_u16_fix_cond_monopoly, iters) {
}
BENCHMARK(atomic_fetch_set_u16_fix_cond_fallback, iters) {
auto op = atomic_util_access::atomic_fetch_set_fallback;
auto op = folly::detail::atomic_fetch_set_fallback;
atomic_fetch_op_fix_(uint16_t(0), op, what_constant<what::cond>{}, iters);
}
BENCHMARK(atomic_fetch_set_u16_fix_cond_native, iters) {
auto op = atomic_util_access::atomic_fetch_set;
auto op = folly::atomic_fetch_set;
atomic_fetch_op_fix_(uint16_t(0), op, what_constant<what::cond>{}, iters);
}
......
folly/synchronization/test/AtomicUtilTest.cpp
View file @ d60894bf
......@@ -105,15 +105,6 @@ TEST_F(AtomicCompareExchangeSuccTest, examples) {
}
}
namespace access {
FOLLY_CREATE_FREE_INVOKER_SUITE(atomic_fetch_set, folly);
FOLLY_CREATE_FREE_INVOKER_SUITE(atomic_fetch_reset, folly);
FOLLY_CREATE_FREE_INVOKER_SUITE(atomic_fetch_flip, folly);
FOLLY_CREATE_FREE_INVOKER_SUITE(atomic_fetch_set_fallback, folly::detail);
FOLLY_CREATE_FREE_INVOKER_SUITE(atomic_fetch_reset_fallback, folly::detail);
FOLLY_CREATE_FREE_INVOKER_SUITE(atomic_fetch_flip_fallback, folly::detail);
} // namespace access
namespace {
template <typename I>
......@@ -125,7 +116,7 @@ struct with_seq_cst : private I {
}
};
template <typename Integer, typename Op = access::atomic_fetch_set_fn>
template <typename Integer, typename Op = folly::atomic_fetch_set_fn>
void atomic_fetch_set_basic(Op fetch_set = {}) {
{
auto&& atomic = std::atomic<Integer>{0};
......@@ -165,7 +156,7 @@ void atomic_fetch_set_basic(Op fetch_set = {}) {
}
}
template <typename Integer, typename Op = access::atomic_fetch_reset_fn>
template <typename Integer, typename Op = folly::atomic_fetch_reset_fn>
void atomic_fetch_reset_basic(Op fetch_reset = {}) {
{
auto&& atomic = std::atomic<Integer>{0};
......@@ -188,7 +179,7 @@ void atomic_fetch_reset_basic(Op fetch_reset = {}) {
}
}
template <typename Integer, typename Op = access::atomic_fetch_flip_fn>
template <typename Integer, typename Op = folly::atomic_fetch_flip_fn>
void atomic_fetch_flip_basic(Op fetch_flip = {}) {
{
auto&& atomic = std::atomic<Integer>{0};
......@@ -214,6 +205,8 @@ void atomic_fetch_flip_basic(Op fetch_flip = {}) {
template <typename Integer>
class Atomic {
public:
using value_type = Integer;
Integer fetch_or(
Integer value, std::memory_order = std::memory_order_seq_cst) {
++counts.set;
......@@ -244,7 +237,7 @@ class Atomic {
counts_ counts;
};
template <typename Integer, typename Op = access::atomic_fetch_set_fn>
template <typename Integer, typename Op = folly::atomic_fetch_set_fn>
void atomic_fetch_set_non_std_atomic(Op fetch_set = {}) {
auto atomic = Atomic<Integer>{};
auto& sets = atomic.counts.set;
......@@ -264,7 +257,7 @@ void atomic_fetch_set_non_std_atomic(Op fetch_set = {}) {
EXPECT_EQ(atomic.integer_, 0b101);
}
template <typename Integer, typename Op = access::atomic_fetch_reset_fn>
template <typename Integer, typename Op = folly::atomic_fetch_reset_fn>
void atomic_fetch_reset_non_std_atomic(Op fetch_reset = {}) {
auto atomic = Atomic<Integer>{};
auto& sets = atomic.counts.set;
......@@ -285,7 +278,7 @@ void atomic_fetch_reset_non_std_atomic(Op fetch_reset = {}) {
EXPECT_EQ(atomic.integer_, 0b010);
}
template <typename Integer, typename Op = access::atomic_fetch_flip_fn>
template <typename Integer, typename Op = folly::atomic_fetch_flip_fn>
void atomic_fetch_flip_non_std_atomic(Op fetch_flip = {}) {
auto atomic = Atomic<Integer>{};
auto& sets = atomic.counts.set;
......@@ -354,7 +347,7 @@ TEST_F(AtomicFetchFlipTest, EnsureFetchXorUsed) {
}
TEST_F(AtomicFetchSetTest, FetchSetFallback) {
auto fetch_set = with_seq_cst{access::atomic_fetch_set_fallback};
auto fetch_set = with_seq_cst{folly::detail::atomic_fetch_set_fallback};
atomic_fetch_set_basic<std::uint16_t>(fetch_set);
atomic_fetch_set_basic<std::uint32_t>(fetch_set);
......@@ -368,7 +361,7 @@ TEST_F(AtomicFetchSetTest, FetchSetFallback) {
}
TEST_F(AtomicFetchResetTest, FetchResetFallback) {
auto fetch_reset = with_seq_cst{access::atomic_fetch_reset_fallback};
auto fetch_reset = with_seq_cst{folly::detail::atomic_fetch_reset_fallback};
atomic_fetch_reset_basic<std::uint16_t>(fetch_reset);
atomic_fetch_reset_basic<std::uint32_t>(fetch_reset);
......@@ -382,7 +375,7 @@ TEST_F(AtomicFetchResetTest, FetchResetFallback) {
}
TEST_F(AtomicFetchFlipTest, FetchFlipFallback) {
auto fetch_flip = with_seq_cst{access::atomic_fetch_flip_fallback};
auto fetch_flip = with_seq_cst{folly::detail::atomic_fetch_flip_fallback};
atomic_fetch_flip_basic<std::uint16_t>(fetch_flip);
atomic_fetch_flip_basic<std::uint32_t>(fetch_flip);
......@@ -396,7 +389,7 @@ TEST_F(AtomicFetchFlipTest, FetchFlipFallback) {
}
TEST_F(AtomicFetchSetTest, FetchSetDefault) {
auto fetch_set = access::atomic_fetch_set;
auto fetch_set = folly::atomic_fetch_set;
atomic_fetch_set_basic<std::uint16_t>(fetch_set);
atomic_fetch_set_basic<std::uint32_t>(fetch_set);
......@@ -410,7 +403,7 @@ TEST_F(AtomicFetchSetTest, FetchSetDefault) {
}
TEST_F(AtomicFetchResetTest, FetchResetDefault) {
auto fetch_reset = access::atomic_fetch_reset;
auto fetch_reset = folly::atomic_fetch_reset;
atomic_fetch_reset_basic<std::uint16_t>(fetch_reset);
atomic_fetch_reset_basic<std::uint32_t>(fetch_reset);
......@@ -424,7 +417,7 @@ TEST_F(AtomicFetchResetTest, FetchResetDefault) {
}
TEST_F(AtomicFetchFlipTest, FetchFlipDefault) {
auto fetch_flip = access::atomic_fetch_flip;
auto fetch_flip = folly::atomic_fetch_flip;
atomic_fetch_flip_basic<std::uint16_t>(fetch_flip);
atomic_fetch_flip_basic<std::uint32_t>(fetch_flip);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment