Commit 146b07b5 authored by Aaryaman Sagar's avatar Aaryaman Sagar Committed by Facebook Github Bot

Extend futex to work with non-standard widths

Summary:
Contains a subset of the functions described in p1135r0, with some additions
for timed waiting that essentially extends the futex interface to work with
non-standard futex widths

In the regular 32 bit case, we fall back to the existing folly futex()
implementation.  In all other cases, we use folly::ParkingLot to mimic futex()

Reviewed By: djwatson

Differential Revision: D9381922

fbshipit-source-id: faf84e105e1d44a6dd6034e25440fcb3eb664846
parent fafeb979
......@@ -17,6 +17,7 @@
#pragma once
#include <folly/detail/Futex.h>
#include <folly/synchronization/ParkingLot.h>
namespace folly {
namespace detail {
......@@ -52,20 +53,23 @@ typename TargetClock::time_point time_point_conv(
* because ADL lookup finds the definitions of these functions when you pass
* the relevant arguments
*/
int futexWakeImpl(Futex<std::atomic>* futex, int count, uint32_t wakeMask);
int futexWakeImpl(
const Futex<std::atomic>* futex,
int count,
uint32_t wakeMask);
FutexResult futexWaitImpl(
Futex<std::atomic>* futex,
const Futex<std::atomic>* futex,
uint32_t expected,
std::chrono::system_clock::time_point const* absSystemTime,
std::chrono::steady_clock::time_point const* absSteadyTime,
uint32_t waitMask);
int futexWakeImpl(
Futex<EmulatedFutexAtomic>* futex,
const Futex<EmulatedFutexAtomic>* futex,
int count,
uint32_t wakeMask);
FutexResult futexWaitImpl(
Futex<EmulatedFutexAtomic>* futex,
const Futex<EmulatedFutexAtomic>* futex,
uint32_t expected,
std::chrono::system_clock::time_point const* absSystemTime,
std::chrono::steady_clock::time_point const* absSteadyTime,
......@@ -92,20 +96,21 @@ futexWaitImpl(
}
template <typename Futex>
FutexResult futexWait(Futex* futex, uint32_t expected, uint32_t waitMask) {
FutexResult
futexWait(const Futex* futex, uint32_t expected, uint32_t waitMask) {
auto rv = futexWaitImpl(futex, expected, nullptr, nullptr, waitMask);
assert(rv != FutexResult::TIMEDOUT);
return rv;
}
template <typename Futex>
int futexWake(Futex* futex, int count, uint32_t wakeMask) {
int futexWake(const Futex* futex, int count, uint32_t wakeMask) {
return futexWakeImpl(futex, count, wakeMask);
}
template <typename Futex, class Clock, class Duration>
FutexResult futexWaitUntil(
Futex* futex,
const Futex* futex,
uint32_t expected,
std::chrono::time_point<Clock, Duration> const& deadline,
uint32_t waitMask) {
......
......@@ -56,7 +56,7 @@ namespace {
# define FUTEX_CLOCK_REALTIME 256
#endif
int nativeFutexWake(void* addr, int count, uint32_t wakeMask) {
int nativeFutexWake(const void* addr, int count, uint32_t wakeMask) {
int rv = syscall(__NR_futex,
addr, /* addr1 */
FUTEX_WAKE_BITSET | FUTEX_PRIVATE_FLAG, /* op */
......@@ -98,7 +98,7 @@ timeSpecFromTimePoint(time_point<Clock> absTime)
}
FutexResult nativeFutexWaitImpl(
void* addr,
const void* addr,
uint32_t expected,
system_clock::time_point const* absSystemTime,
steady_clock::time_point const* absSteadyTime,
......@@ -162,7 +162,7 @@ FutexResult nativeFutexWaitImpl(
using Lot = ParkingLot<uint32_t>;
Lot parkingLot;
int emulatedFutexWake(void* addr, int count, uint32_t waitMask) {
int emulatedFutexWake(const void* addr, int count, uint32_t waitMask) {
int woken = 0;
parkingLot.unpark(addr, [&](const uint32_t& mask) {
if ((mask & waitMask) == 0) {
......@@ -185,8 +185,8 @@ FutexResult emulatedFutexWaitImpl(
steady_clock::time_point const* absSteadyTime,
uint32_t waitMask) {
static_assert(
std::is_same<F, Futex<std::atomic>>::value ||
std::is_same<F, Futex<EmulatedFutexAtomic>>::value,
std::is_same<F, const Futex<std::atomic>>::value ||
std::is_same<F, const Futex<EmulatedFutexAtomic>>::value,
"Type F must be either Futex<std::atomic> or Futex<EmulatedFutexAtomic>");
ParkResult res;
if (absSystemTime) {
......@@ -224,7 +224,10 @@ FutexResult emulatedFutexWaitImpl(
/////////////////////////////////
// Futex<> overloads
int futexWakeImpl(Futex<std::atomic>* futex, int count, uint32_t wakeMask) {
int futexWakeImpl(
const Futex<std::atomic>* futex,
int count,
uint32_t wakeMask) {
#ifdef __linux__
return nativeFutexWake(futex, count, wakeMask);
#else
......@@ -233,14 +236,14 @@ int futexWakeImpl(Futex<std::atomic>* futex, int count, uint32_t wakeMask) {
}
int futexWakeImpl(
Futex<EmulatedFutexAtomic>* futex,
const Futex<EmulatedFutexAtomic>* futex,
int count,
uint32_t wakeMask) {
return emulatedFutexWake(futex, count, wakeMask);
}
FutexResult futexWaitImpl(
Futex<std::atomic>* futex,
const Futex<std::atomic>* futex,
uint32_t expected,
system_clock::time_point const* absSystemTime,
steady_clock::time_point const* absSteadyTime,
......@@ -255,7 +258,7 @@ FutexResult futexWaitImpl(
}
FutexResult futexWaitImpl(
Futex<EmulatedFutexAtomic>* futex,
const Futex<EmulatedFutexAtomic>* futex,
uint32_t expected,
system_clock::time_point const* absSystemTime,
steady_clock::time_point const* absSteadyTime,
......
......@@ -56,7 +56,8 @@ using Futex = Atom<std::uint32_t>;
* other return (signal, this->load() != expected, or spurious wakeup).
*/
template <typename Futex>
FutexResult futexWait(Futex* futex, uint32_t expected, uint32_t waitMask = -1);
FutexResult
futexWait(const Futex* futex, uint32_t expected, uint32_t waitMask = -1);
/**
* Similar to futexWait but also accepts a deadline until when the wait call
......@@ -73,7 +74,7 @@ template <
class Clock,
class Duration = typename Clock::duration>
FutexResult futexWaitUntil(
Futex* futex,
const Futex* futex,
uint32_t expected,
std::chrono::time_point<Clock, Duration> const& deadline,
uint32_t waitMask = -1);
......@@ -89,7 +90,7 @@ FutexResult futexWaitUntil(
*/
template <typename Futex>
int futexWake(
Futex* futex,
const Futex* futex,
int count = std::numeric_limits<int>::max(),
uint32_t wakeMask = -1);
......
/*
* Copyright 2004-present Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include <folly/detail/Futex.h>
#include <folly/synchronization/ParkingLot.h>
#include <condition_variable>
#include <cstdint>
namespace folly {
namespace detail {
namespace atomic_notification {
/**
* We use Futex<std::atomic> as the alias that has the lowest performance
* overhead with respect to atomic notifications. Assert that
* atomic_uint_fast_wait_t is the same as Futex<std::atomic>
*/
static_assert(std::is_same<atomic_uint_fast_wait_t, Futex<std::atomic>>{}, "");
/**
* Implementation and specializations for the atomic_wait() family of
* functions
*/
inline std::cv_status toCvStatus(FutexResult result) {
return (result == FutexResult::TIMEDOUT) ? std::cv_status::timeout
: std::cv_status::no_timeout;
}
inline std::cv_status toCvStatus(ParkResult result) {
return (result == ParkResult::Timeout) ? std::cv_status::timeout
: std::cv_status::no_timeout;
}
// ParkingLot instantiation for futex management
extern ParkingLot<std::uint32_t> parkingLot;
template <template <typename...> class Atom, typename... Args>
void atomic_wait_impl(
const Atom<std::uint32_t, Args...>* atomic,
std::uint32_t expected) {
futexWait(atomic, expected);
return;
}
template <template <typename...> class Atom, typename Integer, typename... Args>
void atomic_wait_impl(const Atom<Integer, Args...>* atomic, Integer expected) {
static_assert(!std::is_same<Integer, std::uint32_t>{}, "");
parkingLot.park(
atomic, -1, [&] { return atomic->load() == expected; }, [] {});
}
template <
template <typename...> class Atom,
typename... Args,
typename Clock,
typename Duration>
std::cv_status atomic_wait_until_impl(
const Atom<std::uint32_t, Args...>* atomic,
std::uint32_t expected,
const std::chrono::time_point<Clock, Duration>& deadline) {
return toCvStatus(futexWaitUntil(atomic, expected, deadline));
}
template <
template <typename...> class Atom,
typename Integer,
typename... Args,
typename Clock,
typename Duration>
std::cv_status atomic_wait_until_impl(
const Atom<Integer, Args...>* atomic,
Integer expected,
const std::chrono::time_point<Clock, Duration>& deadline) {
static_assert(!std::is_same<Integer, std::uint32_t>{}, "");
return toCvStatus(parkingLot.park_until(
atomic, -1, [&] { return atomic->load() == expected; }, [] {}, deadline));
}
template <template <typename...> class Atom, typename... Args>
void atomic_notify_one_impl(const Atom<std::uint32_t, Args...>* atomic) {
futexWake(atomic, 1);
return;
}
template <template <typename...> class Atom, typename Integer, typename... Args>
void atomic_notify_one_impl(const Atom<Integer, Args...>* atomic) {
static_assert(!std::is_same<Integer, std::uint32_t>{}, "");
parkingLot.unpark(atomic, [&](const auto& data) {
assert(data == std::numeric_limits<std::uint32_t>::max());
return UnparkControl::RemoveBreak;
});
}
template <template <typename...> class Atom, typename Integer, typename... Args>
void atomic_notify_all_impl(const Atom<std::uint32_t, Args...>* atomic) {
futexWake(atomic);
return;
}
template <template <typename...> class Atom, typename Integer, typename... Args>
void atomic_notify_all_impl(const Atom<Integer, Args...>* atomic) {
static_assert(!std::is_same<Integer, std::uint32_t>{}, "");
parkingLot.unpark(atomic, [&](const auto& data) {
assert(data == std::numeric_limits<std::uint32_t>::max());
return UnparkControl::RemoveContinue;
});
}
} // namespace atomic_notification
} // namespace detail
template <typename Integer>
void atomic_wait(const std::atomic<Integer>* atomic, Integer expected) {
detail::atomic_notification::atomic_wait_impl(atomic, expected);
}
template <typename Integer, typename Clock, typename Duration>
std::cv_status atomic_wait_until(
const std::atomic<Integer>* atomic,
Integer expected,
const std::chrono::time_point<Clock, Duration>& deadline) {
return detail::atomic_notification::atomic_wait_until_impl(
atomic, expected, deadline);
}
template <typename Integer>
void atomic_notify_one(const std::atomic<Integer>* atomic) {
detail::atomic_notification::atomic_notify_one_impl(atomic);
}
template <typename Integer>
void atomic_notify_all(const std::atomic<Integer>* atomic) {
detail::atomic_notification::atomic_notify_all_impl(atomic);
}
} // namespace folly
/*
* Copyright 2018-present Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <folly/synchronization/AtomicNotification.h>
#include <cstdint>
namespace folly {
namespace detail {
namespace atomic_notification {
// ParkingLot instance used for the atomic_wait() family of functions
//
// This has been defined as a static object (as opposed to allocated to avoid
// destruction order problems) because of possible uses coming from
// allocation-sensitive contexts.
ParkingLot<std::uint32_t> parkingLot;
} // namespace atomic_notification
} // namespace detail
} // namespace folly
/*
* Copyright 2004-present Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include <atomic>
#include <condition_variable>
namespace folly {
/**
* The behavior of the atomic_wait() family of functions is semantically
* identical to futex(). Correspondingly, calling atomic_notify_one(),
* atomic_notify_all() is identical to futexWake() with 1 and
* std::numeric_limits<int>::max() respectively
*
* The difference here compared to the futex API above is that it works with
* all types of atomic widths. When a 32 bit atomic integer is used, the
* implementation falls back to using futex() if possible, and the
* compatibility implementation for non-linux systems otherwise. For all
* other integer widths, the compatibility implementation is used
*
* The templating of this API is changed from the standard in the following
* ways
*
* - At the time of writing, libstdc++'s implementation of std::atomic<> does
* not include the value_type alias. So we rely on the atomic type being a
* template class such that the first type is the underlying value type
* - The Atom parameter allows this API to be compatible with
* DeterministicSchedule testing.
* - atomic_wait_until() does not exist in the linked paper, the version here
* is identical to futexWaitUntil() and returns std::cv_status
*/
// mimic: std::atomic_wait, p1135r0
template <typename Integer>
void atomic_wait(const std::atomic<Integer>* atomic, Integer expected);
template <typename Integer, typename Clock, typename Duration>
std::cv_status atomic_wait_until(
const std::atomic<Integer>* atomic,
Integer expected,
const std::chrono::time_point<Clock, Duration>& deadline);
// mimic: std::atomic_notify_one, p1135r0
template <typename Integer>
void atomic_notify_one(const std::atomic<Integer>* atomic);
// mimic: std::atomic_notify_all, p1135r0
template <typename Integer>
void atomic_notify_all(const std::atomic<Integer>* atomic);
// mimic: std::atomic_uint_fast_wait_t
using atomic_uint_fast_wait_t = std::atomic<std::uint32_t>;
} // namespace folly
#include <folly/synchronization/AtomicNotification-inl.h>
/*
* Copyright 2018-present Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <folly/synchronization/AtomicNotification.h>
#include <folly/portability/GTest.h>
#include <thread>
using namespace std::literals;
namespace folly {
namespace {
template <typename Integer>
void run_atomic_wait_basic() {
auto&& atomic = std::atomic<Integer>{0};
auto&& one = std::thread{[&]() {
while (true) {
atomic_wait(&atomic, Integer{0});
if (atomic.load() == 1) {
break;
}
}
}};
atomic.store(1);
atomic_notify_one(&atomic);
one.join();
}
template <typename Integer>
void run_atomic_wait_until_with_timeout() {
auto&& atomic = std::atomic<Integer>{0};
auto&& one = std::thread{[&]() {
auto deadline = std::chrono::steady_clock::now() + 10ms;
while (true) {
auto result = atomic_wait_until(&atomic, Integer{0}, deadline);
// Any sort of spurious wakeup caused due to aliasing should not be
// changing the value in the futex word in proper usage, so we can
// assert that the value should remain unchanged
EXPECT_TRUE(!atomic.load());
if (result == std::cv_status::timeout) {
EXPECT_TRUE(std::chrono::steady_clock::now() >= deadline);
break;
}
}
}};
one.join();
}
template <typename Integer>
void run_atomic_wait_until_with_notification() {
auto&& atomic = std::atomic<Integer>{0};
auto&& one = std::thread{[&]() {
while (true) {
auto result = atomic_wait_until(
&atomic, Integer{0}, std::chrono::steady_clock::time_point::max());
// note that it is safe to check if we returned from the
// atomic_wait_until() call due to a timeout, futex aliasing can cause
// spurious wakeups due to address reuse, but will not cause spurious
// timeouts, since a futex word has only one timeout on the futex queue,
// and does not inherit timeout from a previous futex at the same
// address
EXPECT_TRUE(result != std::cv_status::timeout);
break;
}
EXPECT_EQ(atomic.load(), 1);
}};
atomic.store(1);
atomic_notify_one(&atomic);
one.join();
}
class SimpleBaton {
public:
void wait() {
auto lck = std::unique_lock<std::mutex>{mutex_};
while (!signalled_) {
cv_.wait(lck);
}
EXPECT_TRUE(signalled_);
}
bool try_wait() {
auto lck = std::unique_lock<std::mutex>{mutex_};
return signalled_;
}
void post() {
auto lck = std::unique_lock<std::mutex>{mutex_};
signalled_ = true;
cv_.notify_one();
}
private:
std::mutex mutex_;
std::condition_variable cv_;
bool signalled_{false};
};
template <typename Integer>
void run_atomic_aliasing() {
auto&& atomic = folly::Optional<std::atomic<Integer>>{folly::in_place, 0};
auto&& one = SimpleBaton{};
auto&& two = SimpleBaton{};
auto threadOne = std::thread{[&]() {
while (true) {
one.wait();
atomic_wait(atomic.get_pointer(), Integer{0});
if (atomic->load() == 1) {
break;
}
}
}};
atomic->store(1);
one.post();
threadOne.join();
// reset the atomic variable
atomic.reset();
atomic.emplace(0);
auto threadTwo = std::thread{[&]() {
atomic_wait(atomic.get_pointer(), Integer{0});
two.post();
}};
while (!two.try_wait()) {
atomic_notify_one(atomic.get_pointer());
}
threadTwo.join();
}
} // namespace
TEST(AtomicWait, Basic) {
run_atomic_wait_basic<std::uint32_t>();
}
TEST(AtomicWait, BasicNonStandardWidths) {
run_atomic_wait_basic<std::uint8_t>();
run_atomic_wait_basic<std::uint16_t>();
run_atomic_wait_basic<std::uint64_t>();
}
TEST(AtomicWait, AtomicWaitUntilTimeout) {
run_atomic_wait_until_with_timeout<std::uint32_t>();
}
TEST(AtomicWait, AtomicWaitUntilTimeoutNonStandardWidths) {
run_atomic_wait_until_with_timeout<std::uint8_t>();
run_atomic_wait_until_with_timeout<std::uint16_t>();
run_atomic_wait_until_with_timeout<std::uint64_t>();
}
TEST(AtomicWait, AtomicWaitUntilNotified) {
run_atomic_wait_until_with_notification<std::uint32_t>();
}
TEST(AtomicWait, AtomicWaitUntilNotifiedNonStandardWidths) {
run_atomic_wait_until_with_notification<std::uint8_t>();
run_atomic_wait_until_with_notification<std::uint16_t>();
run_atomic_wait_until_with_notification<std::uint64_t>();
}
TEST(AtomicWait, AtomicWaitAliasing) {
run_atomic_aliasing<std::uint32_t>();
}
TEST(AtomicWait, AtomicWaitAliasingNonStandardWidths) {
run_atomic_aliasing<std::uint8_t>();
run_atomic_aliasing<std::uint16_t>();
run_atomic_aliasing<std::uint64_t>();
}
} // namespace folly
......@@ -37,8 +37,10 @@ thread_local AuxAct DeterministicSchedule::tls_aux_act;
AuxChk DeterministicSchedule::aux_chk;
// access is protected by futexLock
static std::unordered_map<detail::Futex<DeterministicAtomic>*,
std::list<std::pair<uint32_t, bool*>>> futexQueues;
static std::unordered_map<
const detail::Futex<DeterministicAtomic>*,
std::list<std::pair<uint32_t, bool*>>>
futexQueues;
static std::mutex futexLock;
......@@ -303,7 +305,7 @@ void DeterministicSchedule::wait(sem_t* sem) {
}
detail::FutexResult futexWaitImpl(
detail::Futex<DeterministicAtomic>* futex,
const detail::Futex<DeterministicAtomic>* futex,
uint32_t expected,
std::chrono::system_clock::time_point const* absSystemTimeout,
std::chrono::steady_clock::time_point const* absSteadyTimeout,
......@@ -377,7 +379,7 @@ detail::FutexResult futexWaitImpl(
}
int futexWakeImpl(
detail::Futex<test::DeterministicAtomic>* futex,
const detail::Futex<test::DeterministicAtomic>* futex,
int count,
uint32_t wakeMask) {
using namespace test;
......
......@@ -457,11 +457,11 @@ struct DeterministicAtomic {
/* Futex extensions for DeterministicSchedule based Futexes */
int futexWakeImpl(
detail::Futex<test::DeterministicAtomic>* futex,
const detail::Futex<test::DeterministicAtomic>* futex,
int count,
uint32_t wakeMask);
detail::FutexResult futexWaitImpl(
detail::Futex<test::DeterministicAtomic>* futex,
const detail::Futex<test::DeterministicAtomic>* futex,
uint32_t expected,
std::chrono::system_clock::time_point const* absSystemTime,
std::chrono::steady_clock::time_point const* absSteadyTime,
......
......@@ -18,6 +18,7 @@
#include <folly/test/DeterministicSchedule.h>
#include <chrono>
#include <condition_variable>
#include <functional>
#include <ratio>
#include <thread>
......
......@@ -87,18 +87,19 @@ template <typename T>
struct MockAtom : public std::atomic<T> {
explicit MockAtom(T init = 0) : std::atomic<T>(init) {}
MOCK_METHOD2(futexWait, FutexResult(uint32_t, uint32_t));
MOCK_METHOD3(futexWaitUntil,
FutexResult(uint32_t, const MockClock::time_point&, uint32_t));
MOCK_CONST_METHOD2(futexWait, FutexResult(uint32_t, uint32_t));
MOCK_CONST_METHOD3(
futexWaitUntil,
FutexResult(uint32_t, const MockClock::time_point&, uint32_t));
};
FutexResult
futexWait(Futex<MockAtom>* futex, uint32_t expected, uint32_t waitMask) {
futexWait(const Futex<MockAtom>* futex, uint32_t expected, uint32_t waitMask) {
return futex->futexWait(expected, waitMask);
}
template <typename Clock, typename Duration>
FutexResult futexWaitUntil(
Futex<MockAtom>* futex,
const Futex<MockAtom>* futex,
std::uint32_t expected,
std::chrono::time_point<Clock, Duration> const& deadline,
uint32_t waitMask) {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment