Use a larger stack if sigaltstack is too small

Summary:
The signal handler might use nearly 9KB stack for symbolization. That would
cause stack overflow if the signal handler is running on a small alternative
stack set by sigaltstack. We have seen that in Rust programs. The Rust stdlib
sets up [8KB](https://github.com/rust-lang/libc/blob/4bd52f5e3ef5d6c8abdd1fbaf7c9975800a0dc93/src/unix/notbsd/linux/other/b64/x86_64.rs#L505) stacks [using sigaltstack](https://github.com/rust-lang/rust/blob/55b54a999bcdb0b1c1f42b6e1ae670beb0717086/src/libstd/sys/unix/stack_overflow.rs#L173) for all threads created by Rust on x64 Linux.

Talked with jsgf, we'd still like to get a nice symbolized stack trace if
segfault happens in a binary including both Rust and C++ code, since it's
more likely the C++ part being wrong, so C++ symbol demangling is more
important than Rust demangling.

This diff detects the small sigaltstack case, then allocates larger stack for
symbolization. Note the stack manipulation logic is not signal-safe, so it's
not used for non-small-sigaltstack cases. Programs not using sigaltstack, or
use sigaltstack with >9KB stacks won't be affected.

As we're here, teach the signal handler to save and restore errno. Also add
a note about golang's SA_ONSTACK requirement.

Reviewed By: luciang

Differential Revision: D9846902

fbshipit-source-id: f0dc81b1c1249d8a724e112906e0ff0b14c14ea9

folly/experimental/symbolizer/SignalHandler.cpp

@@ -23,6 +23,7 @@
 
 #include <algorithm>
 #include <atomic>
+#include <cerrno>
 #include <ctime>
 #include <mutex>
 #include <vector>
@@ -128,7 +129,9 @@ void callPreviousSignalHandler(int signum) {
 // in our signal handler at a time.
 //
 // Leak it so we don't have to worry about destruction order
-SafeStackTracePrinter* gStackTracePrinter = new SafeStackTracePrinter();
+//
+// Initialized by installFatalSignalHandler
+SafeStackTracePrinter* gStackTracePrinter;
 
 void printDec(uint64_t val) {
   char buf[20];
@@ -418,8 +421,10 @@ void innerSignalHandler(int signum, siginfo_t* info, void* /* uctx */) {
 }
 
 void signalHandler(int signum, siginfo_t* info, void* uctx) {
+  int savedErrno = errno;
   SCOPE_EXIT {
     flush();
+    errno = savedErrno;
   };
   innerSignalHandler(signum, info, uctx);
 
@@ -442,6 +447,22 @@ namespace {
 
 std::atomic<bool> gAlreadyInstalled;
 
+// Small sigaltstack size threshold.
+// 8931 is known to cause the signal handler to stack overflow during
+// symbolization even for a simple one-liner "kill(getpid(), SIGTERM)".
+const size_t kSmallSigAltStackSize = 8931;
+
+bool isSmallSigAltStackEnabled() {
+  stack_t ss;
+  if (sigaltstack(nullptr, &ss) != 0) {
+    return false;
+  }
+  if ((ss.ss_flags & SS_DISABLE) != 0) {
+    return false;
+  }
+  return ss.ss_size <= kSmallSigAltStackSize;
+}
+
 } // namespace
 
 void installFatalSignalHandler() {
@@ -450,14 +471,32 @@ void installFatalSignalHandler() {
     return;
   }
 
+  // If a small sigaltstack is enabled (ex. Rust stdlib might use sigaltstack
+  // to set a small stack), the default SafeStackTracePrinter would likely
+  // stack overflow. Replace it with the unsafe self-allocate printer.
+  bool useUnsafePrinter = isSmallSigAltStackEnabled();
+  if (useUnsafePrinter) {
+    gStackTracePrinter = new UnsafeSelfAllocateStackTracePrinter();
+  } else {
+    gStackTracePrinter = new SafeStackTracePrinter();
+  }
+
   struct sigaction sa;
   memset(&sa, 0, sizeof(sa));
-  sigemptyset(&sa.sa_mask);
+  if (useUnsafePrinter) {
+    // The signal handler is not async-signal-safe. Block all signals to
+    // make it safer. But it's still unsafe.
+    sigfillset(&sa.sa_mask);
+  } else {
+    sigemptyset(&sa.sa_mask);
+  }
   // By default signal handlers are run on the signaled thread's stack.
   // In case of stack overflow running the SIGSEGV signal handler on
   // the same stack leads to another SIGSEGV and crashes the program.
   // Use SA_ONSTACK, so alternate stack is used (only if configured via
   // sigaltstack).
+  // Golang also requires SA_ONSTACK. See:
+  // https://golang.org/pkg/os/signal/#hdr-Go_programs_that_use_cgo_or_SWIG
   sa.sa_flags |= SA_SIGINFO | SA_ONSTACK;
   sa.sa_sigaction = &signalHandler;
 

folly/experimental/symbolizer/Symbolizer.cpp

@@ -17,6 +17,7 @@
 #include <folly/experimental/symbolizer/Symbolizer.h>
 
 #include <link.h>
+#include <ucontext.h>
 
 #include <climits>
 #include <cstdio>
@@ -37,6 +38,7 @@
 #include <folly/experimental/symbolizer/Dwarf.h>
 #include <folly/experimental/symbolizer/Elf.h>
 #include <folly/experimental/symbolizer/LineReader.h>
+#include <folly/portability/SysMman.h>
 #include <folly/portability/Unistd.h>
 
 /*
@@ -420,6 +422,24 @@ void SafeStackTracePrinter::flush() {
   fsyncNoInt(fd_);
 }
 
+void SafeStackTracePrinter::printSymbolizedStackTrace() {
+  // This function might run on an alternative stack allocated by
+  // UnsafeSelfAllocateStackTracePrinter. Capturing a stack from
+  // here is probably wrong.
+
+  // Do our best to populate location info, process is going to terminate,
+  // so performance isn't critical.
+  Symbolizer symbolizer(&elfCache_, Dwarf::LocationInfoMode::FULL);
+  symbolizer.symbolize(*addresses_);
+
+  // Skip the top 2 frames captured by printStackTrace:
+  // getStackTraceSafe
+  // SafeStackTracePrinter::printStackTrace (captured stack)
+  //
+  // Leaving signalHandler on the stack for clarity, I think.
+  printer_.println(*addresses_, 2);
+}
+
 void SafeStackTracePrinter::printStackTrace(bool symbolize) {
   SCOPE_EXIT {
     flush();
@@ -429,17 +449,7 @@ void SafeStackTracePrinter::printStackTrace(bool symbolize) {
   if (!getStackTraceSafe(*addresses_)) {
     print("(error retrieving stack trace)\n");
   } else if (symbolize) {
-    // Do our best to populate location info, process is going to terminate,
-    // so performance isn't critical.
-    Symbolizer symbolizer(&elfCache_, Dwarf::LocationInfoMode::FULL);
-    symbolizer.symbolize(*addresses_);
-
-    // Skip the top 2 frames:
-    // getStackTraceSafe
-    // SafeStackTracePrinter::printStackTrace (here)
-    //
-    // Leaving signalHandler on the stack for clarity, I think.
-    printer_.println(*addresses_, 2);
+    printSymbolizedStackTrace();
   } else {
     print("(safe mode, symbolizer not available)\n");
     AddressFormatter formatter;
@@ -496,5 +506,95 @@ void FastStackTracePrinter::flush() {
   printer_->flush();
 }
 
+// Stack utilities used by UnsafeSelfAllocateStackTracePrinter
+namespace {
+// Size of mmap-allocated stack. Not to confuse with sigaltstack.
+const size_t kMmapStackSize = 1 * 1024 * 1024;
+
+using MmapPtr = std::unique_ptr<char, void (*)(char*)>;
+
+MmapPtr getNull() {
+  return MmapPtr(nullptr, [](char*) {});
+}
+
+// Assign a mmap-allocated stack to oucp.
+// Return a non-empty smart pointer on success.
+MmapPtr allocateStack(ucontext_t* oucp, size_t pageSize) {
+  MmapPtr p(
+      (char*)mmap(
+          nullptr,
+          kMmapStackSize,
+          PROT_WRITE | PROT_READ,
+          MAP_ANONYMOUS | MAP_PRIVATE,
+          /* fd */ -1,
+          /* offset */ 0),
+      [](char* addr) {
+        // Usually runs inside a fatal signal handler.
+        // Error handling is skipped.
+        munmap(addr, kMmapStackSize);
+      });
+
+  if (!p) {
+    return getNull();
+  }
+
+  // Prepare read-only guard pages on both ends
+  if (pageSize * 2 >= kMmapStackSize) {
+    return getNull();
+  }
+  size_t upperBound = ((kMmapStackSize - 1) / pageSize) * pageSize;
+  if (mprotect(p.get(), pageSize, PROT_NONE) != 0) {
+    return getNull();
+  }
+  if (mprotect(p.get() + upperBound, kMmapStackSize - upperBound, PROT_NONE) !=
+      0) {
+    return getNull();
+  }
+
+  oucp->uc_stack.ss_sp = p.get() + pageSize;
+  oucp->uc_stack.ss_size = upperBound - pageSize;
+  oucp->uc_stack.ss_flags = 0;
+
+  return p;
+}
+
+} // namespace
+
+void UnsafeSelfAllocateStackTracePrinter::printSymbolizedStackTrace() {
+  if (pageSizeUnchecked_ <= 0) {
+    return;
+  }
+
+  ucontext_t cur;
+  memset(&cur, 0, sizeof(cur));
+  ucontext_t alt;
+  memset(&alt, 0, sizeof(alt));
+
+  if (getcontext(&alt) != 0) {
+    return;
+  }
+  alt.uc_link = &cur;
+
+  MmapPtr p = allocateStack(&alt, (size_t)pageSizeUnchecked_);
+  if (!p) {
+    return;
+  }
+
+  auto contextStart = [](UnsafeSelfAllocateStackTracePrinter* that) {
+    that->SafeStackTracePrinter::printSymbolizedStackTrace();
+  };
+
+  makecontext(
+      &alt,
+      (void (*)())(void (*)(UnsafeSelfAllocateStackTracePrinter*))(
+          contextStart),
+      /* argc */ 1,
+      /* arg */ this);
+  // NOTE: swapcontext is not async-signal-safe
+  if (swapcontext(&cur, &alt) != 0) {
+    return;
+  }
+}
+
 } // namespace symbolizer
 } // namespace folly

folly/experimental/symbolizer/Symbolizer.h

@@ -356,6 +356,8 @@ class SafeStackTracePrinter {
       size_t minSignalSafeElfCacheSize = kDefaultMinSignalSafeElfCacheSize,
       int fd = STDERR_FILENO);
 
+  virtual ~SafeStackTracePrinter() {}
+
   /**
    * Only allocates on the stack and is signal-safe but not thread-safe.  Don't
    * call printStackTrace() on the same StackTracePrinter object from multiple
@@ -373,6 +375,9 @@ class SafeStackTracePrinter {
   // Flush printer_, also fsync, in case we're about to crash again...
   void flush();
 
+ protected:
+  virtual void printSymbolizedStackTrace();
+
  private:
   static constexpr size_t kMaxStackTraceDepth = 100;
 
@@ -415,5 +420,22 @@ class FastStackTracePrinter {
   const std::unique_ptr<SymbolizePrinter> printer_;
   Symbolizer symbolizer_;
 };
+
+/**
+ * Use this class in rare situations where signal handlers are running in a
+ * tiny stack specified by sigaltstack.
+ *
+ * This is neither thread-safe nor signal-safe. However, it can usually print
+ * something useful while SafeStackTracePrinter would stack overflow.
+ *
+ * Signal handlers would need to block other signals to make this safer.
+ * Note it's still unsafe even with that.
+ */
+class UnsafeSelfAllocateStackTracePrinter : public SafeStackTracePrinter {
+ protected:
+  void printSymbolizedStackTrace() override;
+  const long pageSizeUnchecked_ = sysconf(_SC_PAGESIZE);
+};
+
 } // namespace symbolizer
 } // namespace folly

folly/experimental/symbolizer/test/SmallSigAltStackCrash.cpp

+/*
+ * Copyright 2014-present Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <signal.h>
+
+#include <folly/experimental/symbolizer/SignalHandler.h>
+
+namespace {
+std::array<char, 8192> stack;
+}
+
+int main() {
+  stack_t ss;
+  ss.ss_sp = stack.data();
+  ss.ss_size = stack.size();
+  ss.ss_flags = 0;
+  sigaltstack(&ss, nullptr);
+
+  folly::symbolizer::installFatalSignalHandler();
+  __builtin_trap();
+  return 0;
+}