Skip to content

F
folly
Commit 6e0a487c authored by Yedidya Feldblum's avatar Yedidya Feldblum Committed by Facebook Github Bot
Browse files
Options

Optimize Function::operator() codegen 

Summary:
[Folly] Optimize `Function::operator()` codegen for size and speed.

* Avoid translating between values and references for trivially-copyable values.
* Avoid shifting all arguments to make room for the function object address.

In the optimal case, the codegen for calling a `Function` with many arguments translates into just a `jmp`.

See:
* https://github.com/thecppzoo/zoo/commits/master/inc/zoo/AnyCallable.h
* https://github.com/bloomberg/bde/blob/3.38.0.1/groups/bsl/bslstl/bslstl_function.h
* https://github.com/bloomberg/bde/blob/3.38.0.1/groups/bsl/bslmf/bslmf_forwardingtype.h

Given this example code:

```lang=c++,name=check.cpp
extern "C" void check_0(folly::Function<void()>& f) { f(); }
extern "C" void check_1(int i, folly::Function<void(int)>& f) { f(i); }
extern "C" void check_2(int i, int j, folly::Function<void(int, int)>& f) { f(i, j); }
extern "C" void check_3(int i, int j, int k, folly::Function<void(int, int, int)>& f) { f(i, j, k); }
extern "C" void check_4(int i, int j, int k, int l, folly::Function<void(int, int, int, int)>& f) { f(i, j, k, l); }
```

Before:

```name=check.o
0000000000000000 <check_0>:
   0:   ff 67 30                jmp    QWORD PTR [rdi+0x30]
0000000000000000 <check_1>:
   0:   55                      push   rbp
   1:   48 89 f0                mov    rax,rsi
   4:   48 89 e5                mov    rbp,rsp
   7:   48 83 ec 10             sub    rsp,0x10
   b:   89 7d fc                mov    DWORD PTR [rbp-0x4],edi
   e:   48 8d 75 fc             lea    rsi,[rbp-0x4]
  12:   48 89 c7                mov    rdi,rax
  15:   ff 50 30                call   QWORD PTR [rax+0x30]
  18:   c9                      leave
  19:   c3                      ret
0000000000000000 <check_2>:
   0:   55                      push   rbp
   1:   48 89 d0                mov    rax,rdx
   4:   48 89 e5                mov    rbp,rsp
   7:   48 83 ec 10             sub    rsp,0x10
   b:   89 7d f8                mov    DWORD PTR [rbp-0x8],edi
   e:   89 75 fc                mov    DWORD PTR [rbp-0x4],esi
  11:   48 8d 55 fc             lea    rdx,[rbp-0x4]
  15:   48 8d 75 f8             lea    rsi,[rbp-0x8]
  19:   48 89 c7                mov    rdi,rax
  1c:   ff 50 30                call   QWORD PTR [rax+0x30]
  1f:   c9                      leave
  20:   c3                      ret
0000000000000000 <check_3>:
   0:   55                      push   rbp
   1:   48 89 c8                mov    rax,rcx
   4:   48 89 e5                mov    rbp,rsp
   7:   48 83 ec 10             sub    rsp,0x10
   b:   89 7d f4                mov    DWORD PTR [rbp-0xc],edi
   e:   89 75 f8                mov    DWORD PTR [rbp-0x8],esi
  11:   89 55 fc                mov    DWORD PTR [rbp-0x4],edx
  14:   48 8d 4d fc             lea    rcx,[rbp-0x4]
  18:   48 8d 55 f8             lea    rdx,[rbp-0x8]
  1c:   48 8d 75 f4             lea    rsi,[rbp-0xc]
  20:   48 89 c7                mov    rdi,rax
  23:   ff 50 30                call   QWORD PTR [rax+0x30]
  26:   c9                      leave
  27:   c3                      ret
0000000000000000 <check_4>:
   0:   55                      push   rbp
   1:   4c 89 c0                mov    rax,r8
   4:   48 89 e5                mov    rbp,rsp
   7:   48 83 ec 10             sub    rsp,0x10
   b:   89 7d f0                mov    DWORD PTR [rbp-0x10],edi
   e:   89 75 f4                mov    DWORD PTR [rbp-0xc],esi
  11:   89 55 f8                mov    DWORD PTR [rbp-0x8],edx
  14:   89 4d fc                mov    DWORD PTR [rbp-0x4],ecx
  17:   4c 8d 45 fc             lea    r8,[rbp-0x4]
  1b:   48 8d 4d f8             lea    rcx,[rbp-0x8]
  1f:   48 8d 55 f4             lea    rdx,[rbp-0xc]
  23:   48 8d 75 f0             lea    rsi,[rbp-0x10]
  27:   48 89 c7                mov    rdi,rax
  2a:   ff 50 30                call   QWORD PTR [rax+0x30]
  2d:   c9                      leave
  2e:   c3                      ret
```

After:

```name=check.o
0000000000000000 <check_0>:
   0:   ff 67 30                jmp    QWORD PTR [rdi+0x30]
0000000000000000 <check_1>:
   0:   ff 66 30                jmp    QWORD PTR [rsi+0x30]
0000000000000000 <check_2>:
   0:   ff 62 30                jmp    QWORD PTR [rdx+0x30]
0000000000000000 <check_3>:
   0:   ff 61 30                jmp    QWORD PTR [rcx+0x30]
0000000000000000 <check_4>:
   0:   41 ff 60 30             jmp    QWORD PTR [r8+0x30]
```

Reviewed By: luciang

Differential Revision: D17523239

fbshipit-source-id: beed0bae827aad8290e807374e8596f71f98ce99
parent f01cb070
Hide whitespace changes
Inline Side-by-side
Showing with 50 additions and 28 deletions
folly/Function.h
View file @ 6e0a487c
......@@ -306,6 +306,21 @@ template <
!std::is_reference<To>::value || std::is_reference<From>::value>::type>
using SafeResultOf = decltype(static_cast<To>(std::declval<From>()));
#if _MSC_VER
// Need a workaround for MSVC to avoid the inscrutable error:
//
// folly\function.h(...) : fatal error C1001: An internal error has
// occurred in the compiler.
// (compiler file 'f:\dd\vctools\compiler\utc\src\p2\main.c', line 258)
// To work around this problem, try simplifying or changing the program
// near the locations listed above.
template <typename T>
using CallArg = T&&;
#else
template <typename T>
using CallArg = conditional_t<is_trivially_copyable<T>::value, T, T&&>;
#endif
template <typename F, typename R, typename... A>
class FunctionTraitsSharedProxy {
std::shared_ptr<Function<F>> sp_;
......@@ -346,7 +361,7 @@ struct FunctionTraits;
template <typename ReturnType, typename... Args>
struct FunctionTraits<ReturnType(Args...)> {
using Call = ReturnType (*)(Data&, Args&&...);
using Call = ReturnType (*)(CallArg<Args>..., Data&);
using IsConst = std::false_type;
using ConstSignature = ReturnType(Args...) const;
using NonConstSignature = ReturnType(Args...);
......@@ -357,24 +372,24 @@ struct FunctionTraits<ReturnType(Args...)> {
SafeResultOf<CallableResult<std::decay_t<F>&, Args...>, ReturnType>;
template <typename Fun>
static ReturnType callSmall(Data& p, Args&&... args) {
static ReturnType callSmall(CallArg<Args>... args, Data& p) {
return static_cast<ReturnType>((*static_cast<Fun*>(
static_cast<void*>(&p.tiny)))(static_cast<Args&&>(args)...));
}
template <typename Fun>
static ReturnType callBig(Data& p, Args&&... args) {
static ReturnType callBig(CallArg<Args>... args, Data& p) {
return static_cast<ReturnType>(
(*static_cast<Fun*>(p.big))(static_cast<Args&&>(args)...));
}
static ReturnType uninitCall(Data&, Args&&...) {
static ReturnType uninitCall(CallArg<Args>..., Data&) {
throw_exception<std::bad_function_call>();
}
ReturnType operator()(Args... args) {
auto& fn = *static_cast<Function<NonConstSignature>*>(this);
return fn.call_(fn.data_, static_cast<Args&&>(args)...);
return fn.call_(static_cast<Args&&>(args)..., fn.data_);
}
using SharedProxy =
......@@ -383,7 +398,7 @@ struct FunctionTraits<ReturnType(Args...)> {
template <typename ReturnType, typename... Args>
struct FunctionTraits<ReturnType(Args...) const> {
using Call = ReturnType (*)(Data&, Args&&...);
using Call = ReturnType (*)(CallArg<Args>..., Data&);
using IsConst = std::true_type;
using ConstSignature = ReturnType(Args...) const;
using NonConstSignature = ReturnType(Args...);
......@@ -394,24 +409,24 @@ struct FunctionTraits<ReturnType(Args...) const> {
SafeResultOf<CallableResult<const std::decay_t<F>&, Args...>, ReturnType>;
template <typename Fun>
static ReturnType callSmall(Data& p, Args&&... args) {
static ReturnType callSmall(CallArg<Args>... args, Data& p) {
return static_cast<ReturnType>((*static_cast<const Fun*>(
static_cast<void*>(&p.tiny)))(static_cast<Args&&>(args)...));
}
template <typename Fun>
static ReturnType callBig(Data& p, Args&&... args) {
static ReturnType callBig(CallArg<Args>... args, Data& p) {
return static_cast<ReturnType>(
(*static_cast<const Fun*>(p.big))(static_cast<Args&&>(args)...));
}
static ReturnType uninitCall(Data&, Args&&...) {
static ReturnType uninitCall(CallArg<Args>..., Data&) {
throw_exception<std::bad_function_call>();
}
ReturnType operator()(Args... args) const {
auto& fn = *static_cast<const Function<ConstSignature>*>(this);
return fn.call_(fn.data_, static_cast<Args&&>(args)...);
return fn.call_(static_cast<Args&&>(args)..., fn.data_);
}
using SharedProxy =
......@@ -421,7 +436,7 @@ struct FunctionTraits<ReturnType(Args...) const> {
#if FOLLY_HAVE_NOEXCEPT_FUNCTION_TYPE
template <typename ReturnType, typename... Args>
struct FunctionTraits<ReturnType(Args...) noexcept> {
using Call = ReturnType (*)(Data&, Args&&...) noexcept;
using Call = ReturnType (*)(CallArg<Args>..., Data&) noexcept;
using IsConst = std::false_type;
using ConstSignature = ReturnType(Args...) const noexcept;
using NonConstSignature = ReturnType(Args...) noexcept;
......@@ -432,24 +447,24 @@ struct FunctionTraits<ReturnType(Args...) noexcept> {
SafeResultOf<CallableResult<std::decay_t<F>&, Args...>, ReturnType>;
template <typename Fun>
static ReturnType callSmall(Data& p, Args&&... args) noexcept {
static ReturnType callSmall(CallArg<Args>... args, Data& p) noexcept {
return static_cast<ReturnType>((*static_cast<Fun*>(
static_cast<void*>(&p.tiny)))(static_cast<Args&&>(args)...));
}
template <typename Fun>
static ReturnType callBig(Data& p, Args&&... args) noexcept {
static ReturnType callBig(CallArg<Args>... args, Data& p) noexcept {
return static_cast<ReturnType>(
(*static_cast<Fun*>(p.big))(static_cast<Args&&>(args)...));
}
static ReturnType uninitCall(Data&, Args&&...) noexcept {
static ReturnType uninitCall(CallArg<Args>..., Data&) noexcept {
terminate_with<std::bad_function_call>();
}
ReturnType operator()(Args... args) noexcept {
auto& fn = *static_cast<Function<NonConstSignature>*>(this);
return fn.call_(fn.data_, static_cast<Args&&>(args)...);
return fn.call_(static_cast<Args&&>(args)..., fn.data_);
}
using SharedProxy =
......@@ -458,7 +473,7 @@ struct FunctionTraits<ReturnType(Args...) noexcept> {
template <typename ReturnType, typename... Args>
struct FunctionTraits<ReturnType(Args...) const noexcept> {
using Call = ReturnType (*)(Data&, Args&&...) noexcept;
using Call = ReturnType (*)(CallArg<Args>..., Data&) noexcept;
using IsConst = std::true_type;
using ConstSignature = ReturnType(Args...) const noexcept;
using NonConstSignature = ReturnType(Args...) noexcept;
......@@ -469,24 +484,24 @@ struct FunctionTraits<ReturnType(Args...) const noexcept> {
SafeResultOf<CallableResult<const std::decay_t<F>&, Args...>, ReturnType>;
template <typename Fun>
static ReturnType callSmall(Data& p, Args&&... args) noexcept {
static ReturnType callSmall(CallArg<Args>... args, Data& p) noexcept {
return static_cast<ReturnType>((*static_cast<const Fun*>(
static_cast<void*>(&p.tiny)))(static_cast<Args&&>(args)...));
}
template <typename Fun>
static ReturnType callBig(Data& p, Args&&... args) noexcept {
static ReturnType callBig(CallArg<Args>... args, Data& p) noexcept {
return static_cast<ReturnType>(
(*static_cast<const Fun*>(p.big))(static_cast<Args&&>(args)...));
}
static ReturnType uninitCall(Data&, Args&&...) noexcept {
static ReturnType uninitCall(CallArg<Args>..., Data&) noexcept {
throw_exception<std::bad_function_call>();
}
ReturnType operator()(Args... args) const noexcept {
auto& fn = *static_cast<const Function<ConstSignature>*>(this);
return fn.call_(fn.data_, static_cast<Args&&>(args)...);
return fn.call_(static_cast<Args&&>(args)..., fn.data_);
}
using SharedProxy =
......@@ -913,14 +928,17 @@ class FunctionRef;
template <typename ReturnType, typename... Args>
class FunctionRef<ReturnType(Args...)> final {
using Call = ReturnType (*)(void*, Args&&...);
template <typename Arg>
using CallArg = detail::function::CallArg<Arg>;
using Call = ReturnType (*)(CallArg<Args>..., void*);
static ReturnType uninitCall(void*, Args&&...) {
static ReturnType uninitCall(CallArg<Args>..., void*) {
throw_exception<std::bad_function_call>();
}
template <typename Fun>
static ReturnType call(void* object, Args&&... args) {
static ReturnType call(CallArg<Args>... args, void* object) {
using Pointer = std::add_pointer_t<Fun>;
return static_cast<ReturnType>(invoke(
static_cast<Fun&&>(*static_cast<Pointer>(object)),
......@@ -965,7 +983,7 @@ class FunctionRef<ReturnType(Args...)> final {
call_(&FunctionRef::template call<Fun>) {}
ReturnType operator()(Args... args) const {
return call_(object_, static_cast<Args&&>(args)...);
return call_(static_cast<Args&&>(args)..., object_);
}
constexpr explicit operator bool() const noexcept {
......
folly/synchronization/detail/InlineFunctionRef.h
View file @ 6e0a487c
......@@ -16,6 +16,7 @@
#pragma once
#include <folly/Function.h>
#include <folly/Traits.h>
#include <folly/Utility.h>
#include <folly/functional/Invoke.h>
......@@ -67,9 +68,12 @@ class InlineFunctionRef;
template <typename ReturnType, typename... Args, std::size_t Size>
class InlineFunctionRef<ReturnType(Args...), Size> {
template <typename Arg>
using CallArg = function::CallArg<Arg>;
using Storage =
std::aligned_storage_t<Size - sizeof(uintptr_t), sizeof(uintptr_t)>;
using Call = ReturnType (*)(const Storage&, Args&&...);
using Call = ReturnType (*)(CallArg<Args>..., const Storage&);
struct InSituTag {};
struct RefTag {};
......@@ -149,7 +153,7 @@ class InlineFunctionRef<ReturnType(Args...), Size> {
* appropriate casting.
*/
ReturnType operator()(Args... args) const {
return call_(storage_, static_cast<Args&&>(args)...);
return call_(static_cast<Args&&>(args)..., storage_);
}
/**
......@@ -196,7 +200,7 @@ class InlineFunctionRef<ReturnType(Args...), Size> {
}
template <typename Func>
static ReturnType callInline(const Storage& object, Args&&... args) {
static ReturnType callInline(CallArg<Args>... args, const Storage& object) {
// The only type of pointer allowed is a function pointer, no other
// pointer types are invocable.
static_assert(
......@@ -209,7 +213,7 @@ class InlineFunctionRef<ReturnType(Args...), Size> {
}
template <typename Func>
static ReturnType callPointer(const Storage& object, Args&&... args) {
static ReturnType callPointer(CallArg<Args>... args, const Storage& object) {
// When the function we were instantiated with was not trivial, the given
// pointer points to a pointer, which pointers to the callable. So we
// cast to a pointer and then to the pointee.
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment