A place for compiler hints

Summary:
[Folly] A place for compiler hints in `folly/lang/Hint.h`.

Add long-form names which explicitly reveal intentions:
* `compiler_may_unsafely_assume` v.s. `assume`
* `compiler_may_unsafely_assume_unreachable` v.s. `assume_unreachable`
* `compiler_must_not_elide` v.s. `doNotOptimizeAway`
* `compiler_must_not_predict` v.s. `makeUnpredictable`

Reviewed By: Gownta

Differential Revision: D20240392

fbshipit-source-id: 5f13fafc6a426fcd66fa61465eb1e2ffe15b4187

folly/Benchmark.h

@@ -23,6 +23,7 @@
 #include <folly/ScopeGuard.h>
 #include <folly/Traits.h>
 #include <folly/functional/Invoke.h>
+#include <folly/lang/Hint.h>
 #include <folly/portability/GFlags.h>
 
 #include <cassert>

folly/BenchmarkUtil.h

@@ -16,9 +16,8 @@
 
 #pragma once
 
-#include <type_traits>
-
-#include <folly/Traits.h>
+#include <folly/Portability.h>
+#include <folly/lang/Hint.h>
 
 namespace folly {
 
@@ -35,80 +34,14 @@ namespace folly {
  * benchmark but not in real use cases.
  */
 
-#if defined(_MSC_VER) && !defined(__clang__)
-
-#pragma optimize("", off)
-
-inline void doNotOptimizeDependencySink(const void*) {}
-
-#pragma optimize("", on)
-
 template <class T>
-void doNotOptimizeAway(const T& datum) {
-  doNotOptimizeDependencySink(&datum);
-}
-
-template <typename T>
-void makeUnpredictable(T& datum) {
-  doNotOptimizeDependencySink(&datum);
-}
-
-#else
-
-namespace detail {
-
-template <typename T>
-struct DoNotOptimizeAwayNeedsIndirect {
-  using Decayed = typename std::decay<T>::type;
-
-  // First two constraints ensure it can be an "r" operand.
-  // std::is_pointer check is because callers seem to expect that
-  // doNotOptimizeAway(&x) is equivalent to doNotOptimizeAway(x).
-  constexpr static bool value = !folly::is_trivially_copyable<Decayed>::value ||
-      sizeof(Decayed) > sizeof(long) || std::is_pointer<Decayed>::value;
-};
-
-} // namespace detail
-
-template <typename T>
-auto doNotOptimizeAway(const T& datum) -> typename std::enable_if<
-    !detail::DoNotOptimizeAwayNeedsIndirect<T>::value>::type {
-  // The "r" constraint forces the compiler to make datum available
-  // in a register to the asm block, which means that it must have
-  // computed/loaded it.  We use this path for things that are <=
-  // sizeof(long) (they have to fit), trivial (otherwise the compiler
-  // doesn't want to put them in a register), and not a pointer (because
-  // doNotOptimizeAway(&foo) would otherwise be a foot gun that didn't
-  // necessarily compute foo).
-  //
-  // An earlier version of this method had a more permissive input operand
-  // constraint, but that caused unnecessary variation between clang and
-  // gcc benchmarks.
-  asm volatile("" ::"r"(datum));
-}
-
-template <typename T>
-auto doNotOptimizeAway(const T& datum) -> typename std::enable_if<
-    detail::DoNotOptimizeAwayNeedsIndirect<T>::value>::type {
-  // This version of doNotOptimizeAway tells the compiler that the asm
-  // block will read datum from memory, and that in addition it might read
-  // or write from any memory location.  If the memory clobber could be
-  // separated into input and output that would be preferrable.
-  asm volatile("" ::"m"(datum) : "memory");
-}
-
-template <typename T>
-auto makeUnpredictable(T& datum) -> typename std::enable_if<
-    !detail::DoNotOptimizeAwayNeedsIndirect<T>::value>::type {
-  asm volatile("" : "+r"(datum));
+FOLLY_ALWAYS_INLINE void doNotOptimizeAway(const T& datum) {
+  compiler_must_not_elide(datum);
 }
 
 template <typename T>
-auto makeUnpredictable(T& datum) -> typename std::enable_if<
-    detail::DoNotOptimizeAwayNeedsIndirect<T>::value>::type {
-  asm volatile("" ::"m"(datum) : "memory");
+FOLLY_ALWAYS_INLINE void makeUnpredictable(T& datum) {
+  compiler_must_not_predict(datum);
 }
 
-#endif
-
 } // namespace folly

folly/experimental/coro/detail/Malloc.cpp

@@ -16,7 +16,7 @@
 
 #include <folly/experimental/coro/detail/Malloc.h>
 
-#include <folly/BenchmarkUtil.h>
+#include <folly/lang/Hint.h>
 #include <folly/lang/New.h>
 
 extern "C" {
@@ -27,7 +27,7 @@ void* folly_coro_async_malloc(std::size_t size) {
 
   // Add this after the call to prevent the compiler from
   // turning the call to operator new() into a tailcall.
-  folly::doNotOptimizeAway(p);
+  folly::compiler_must_not_elide(p);
 
   return p;
 }
@@ -38,6 +38,6 @@ void folly_coro_async_free(void* ptr, std::size_t size) {
 
   // Add this after the call to prevent the compiler from
   // turning the call to operator delete() into a tailcall.
-  folly::doNotOptimizeAway(size);
+  folly::compiler_must_not_elide(size);
 }
 } // extern "C"

folly/lang/Assume-inl.h

-/*
- * Copyright (c) Meta Platforms, Inc. and affiliates.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#pragma once
-
-#include <folly/Likely.h>
-#include <folly/Portability.h>
-
-namespace folly {
-
-namespace detail {
-
-[[noreturn]] void assume_terminate();
-
-} // namespace detail
-
-FOLLY_ALWAYS_INLINE void assume(bool cond) {
-  if (kIsDebug) {
-    if (FOLLY_UNLIKELY(!cond)) {
-      detail::assume_terminate();
-    }
-  } else {
-#if defined(__clang__) // Must go first because Clang also defines __GNUC__.
-    __builtin_assume(cond);
-#elif defined(__GNUC__)
-    if (!cond) {
-      __builtin_unreachable();
-    }
-#elif defined(_MSC_VER)
-    __assume(cond);
-#else
-    // Do nothing.
-#endif
-  }
-}
-
-[[noreturn]] FOLLY_ALWAYS_INLINE void assume_unreachable() {
-#if defined(__GNUC__)
-  __builtin_unreachable();
-#elif defined(_MSC_VER)
-  __assume(0);
-#else
-  detail::assume_terminate();
-#endif
-}
-
-} // namespace folly

folly/lang/Assume.cpp

-/*
- * Copyright (c) Meta Platforms, Inc. and affiliates.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <folly/lang/Assume.h>
-
-#include <folly/lang/SafeAssert.h>
-
-namespace folly {
-
-namespace detail {
-
-[[noreturn]] void assume_terminate() {
-  FOLLY_SAFE_CHECK(false, "compiler-hint assumption fails at runtime");
-}
-
-} // namespace detail
-
-} // namespace folly

folly/lang/Assume.h

@@ -17,6 +17,7 @@
 #pragma once
 
 #include <folly/Portability.h>
+#include <folly/lang/Hint.h>
 
 namespace folly {
 
@@ -51,7 +52,9 @@ namespace folly {
  * implemented as a function to force the evaluation of its argument, contrary
  * to the builtin, which cannot used with expressions that have side-effects.
  */
-FOLLY_ALWAYS_INLINE void assume(bool cond);
+FOLLY_ALWAYS_INLINE void assume(bool cond) {
+  compiler_may_unsafely_assume(cond);
+}
 
 /**
  * assume_unreachable() informs the compiler that the statement is not reachable
@@ -62,8 +65,8 @@ FOLLY_ALWAYS_INLINE void assume(bool cond);
  * evaluation of switch/case statements when all the possible values are
  * provably enumerated.
  */
-[[noreturn]] FOLLY_ALWAYS_INLINE void assume_unreachable();
+[[noreturn]] FOLLY_ALWAYS_INLINE void assume_unreachable() {
+  compiler_may_unsafely_assume_unreachable();
+}
 
 } // namespace folly
-
-#include <folly/lang/Assume-inl.h>

folly/lang/Hint-inl.h

+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+namespace folly {
+
+FOLLY_ALWAYS_INLINE void compiler_may_unsafely_assume(bool cond) {
+  FOLLY_SAFE_DCHECK(cond, "compiler-hint assumption fails at runtime");
+#if defined(__clang__)
+  __builtin_assume(cond);
+#elif defined(__GNUC__)
+  if (!cond) {
+    __builtin_unreachable();
+  }
+#elif defined(_MSC_VER)
+  __assume(cond);
+#else
+  while (!cond)
+    ;
+#endif
+}
+
+[[noreturn]] FOLLY_ALWAYS_INLINE void
+compiler_may_unsafely_assume_unreachable() {
+  FOLLY_SAFE_DCHECK(false, "compiler-hint unreachability reached at runtime");
+#if defined(__GNUC__)
+  __builtin_unreachable();
+#elif defined(_MSC_VER)
+  __assume(0);
+#else
+  while (!0)
+    ;
+#endif
+}
+
+#if defined(_MSC_VER) && !defined(__clang__)
+
+namespace detail {
+
+#pragma optimize("", off)
+
+inline void compiler_must_force_sink(void const*) {}
+
+#pragma optimize("", on)
+
+} // namespace detail
+
+template <typename T>
+FOLLY_ALWAYS_INLINE void compiler_must_not_elide_fn::operator()(
+    T const& t) const noexcept {
+  detail::compiler_must_force_sink(&t);
+}
+
+template <typename T>
+FOLLY_ALWAYS_INLINE void compiler_must_not_predict_fn::operator()(
+    T& t) const noexcept {
+  detail::compiler_must_force_sink(&t);
+}
+
+#else
+
+namespace detail {
+
+template <typename T, typename D = std::decay_t<T>>
+using compiler_must_force_indirect = bool_constant<
+    !is_trivially_copyable_v<D> || //
+    sizeof(long) < sizeof(D) || //
+    std::is_pointer<D>::value>;
+
+template <typename T>
+FOLLY_ALWAYS_INLINE void compiler_must_not_elide(T const& t, std::false_type) {
+  // the "r" constraint forces the compiler to make the value available in a
+  // register to the asm block, which means that it must first have been
+  // computed or loaded
+  //
+  // used for small trivial values which the compiler will put into registers
+  //
+  // avoided for pointers to avoid fallout in calling code which mistakenly
+  // applies the hint to the address of a value but not to the value itself
+  asm volatile("" : : "r"(t));
+}
+
+template <typename T>
+FOLLY_ALWAYS_INLINE void compiler_must_not_elide(T const& t, std::true_type) {
+  // tells the compiler that the asm block will read the value from memory,
+  // and that in addition it might read or write from any memory location
+  //
+  // if the memory clobber could be split into input and output, that would be
+  // preferrable
+  asm volatile("" : : "m"(t) : "memory");
+}
+
+template <typename T>
+FOLLY_ALWAYS_INLINE void compiler_must_not_predict(T& t, std::false_type) {
+  asm volatile("" : "+r"(t));
+}
+
+template <typename T>
+FOLLY_ALWAYS_INLINE void compiler_must_not_predict(T& t, std::true_type) {
+  asm volatile("" : : "m"(t) : "memory");
+}
+
+} // namespace detail
+
+template <typename T>
+FOLLY_ALWAYS_INLINE void compiler_must_not_elide_fn::operator()(
+    T const& t) const noexcept {
+  using i = detail::compiler_must_force_indirect<T>;
+  detail::compiler_must_not_elide(t, i{});
+}
+
+template <typename T>
+FOLLY_ALWAYS_INLINE void compiler_must_not_predict_fn::operator()(
+    T& t) const noexcept {
+  using i = detail::compiler_must_force_indirect<T>;
+  detail::compiler_must_not_predict(t, i{});
+}
+
+#endif
+
+} // namespace folly

folly/lang/Hint.h

+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <type_traits>
+
+#include <folly/Portability.h>
+#include <folly/Traits.h>
+#include <folly/lang/SafeAssert.h>
+
+namespace folly {
+
+//  compiler_may_unsafely_assume
+//
+//  Unsafe. Avoid when not absolutely necessary.
+//
+//  Permits the compiler to assume the truth of the provided expression and to
+//  optimize the surrounding code as if that expression were true.
+//
+//  Causes every possible kind of subtle, irreproducible, non-debuggable failure
+//  if the provided expression is ever, for any reason, false: random crashes,
+//  silent memory corruptions, arbitrary code execution, privacy violation, etc.
+//
+//  Must only be used on conditions that are provable internal logic invariants.
+//
+//  Must not be used on conditions which depend on external inputs. For such
+//  cases, an assertion or exception may be used instead.
+void compiler_may_unsafely_assume(bool cond);
+
+//  compiler_may_unsafely_assume_unreachable
+//
+//  Unsafe. Avoid when not absolutely necessary.
+//
+//  Permits the compiler to assume that the given statement cannot be reached
+//  and to optimize the surrounding code accordingly. Permits call sites to omit
+//  return statements in non-void-returning functions without compiler error.
+//
+//  Causes every possible kind of subtle, irreproducible, non-debuggable failure
+//  if the given statement is ever, for any reason, reached: random crashes,
+//  silent memory corruptions, arbitrary code execution, privacy violation, etc.
+//
+//  Must only be used on conditions that are provable internal logic invariants.
+//
+//  Must not be used on conditions which depend on external inputs. For such
+//  cases, an assertion or exception may be used instead.
+[[noreturn]] void compiler_may_unsafely_assume_unreachable();
+
+//  compiler_must_not_elide
+//
+//  Ensures that the referred-to value will be computed even when an optimizing
+//  compiler might otherwise remove the computation. Note: this hint takes its
+//  value paramaeter by reference.
+//
+//  Useful for values that are computed during benchmarking but otherwise are
+//  unused. The compiler tends to do a good job at eliminating unused variables,
+//  which can affect benchmark results, and this hint instructs the compiler to
+//  treat the value as though it were used.
+struct compiler_must_not_elide_fn {
+  template <typename T>
+  FOLLY_ALWAYS_INLINE void operator()(T const& t) const noexcept;
+};
+FOLLY_INLINE_VARIABLE constexpr compiler_must_not_elide_fn
+    compiler_must_not_elide{};
+
+//  compiler_must_not_predict
+//
+//  Ensures that the compiler will not use its knowledge of the referred-to
+//  value to optimize or otherwise shape the following code. Note: this hint
+//  takes its value parameter by reference.
+//
+//  Useful when constant propagation or power reduction is possible in a
+//  benchmark but not in real use cases. Optimizations done to benchmarked code
+//  which cannot be done to real code can affect benchmark results, and this
+//  hint instructs the compiler to treat value which it can predict as though it
+//  were unpredictable.
+struct compiler_must_not_predict_fn {
+  template <typename T>
+  FOLLY_ALWAYS_INLINE void operator()(T& t) const noexcept;
+};
+FOLLY_INLINE_VARIABLE constexpr compiler_must_not_predict_fn
+    compiler_must_not_predict{};
+
+} // namespace folly
+
+#include <folly/lang/Hint-inl.h>

folly/tracing/AsyncStack.cpp

@@ -22,8 +22,8 @@
 
 #include <glog/logging.h>
 
-#include <folly/BenchmarkUtil.h>
 #include <folly/Likely.h>
+#include <folly/lang/Hint.h>
 
 #if defined(__linux__)
 #define FOLLY_ASYNC_STACK_ROOT_USE_PTHREAD 1
@@ -146,7 +146,7 @@ FOLLY_NOINLINE static void* detached_task() noexcept {
 
   // Add this after the call to prevent the compiler from
   // turning the call to get_return_address() into a tailcall.
-  folly::doNotOptimizeAway(p);
+  compiler_must_not_elide(p);
 
   return p;
 }