Option to record precise stack on every N fiber

Summary: title + some logging around

Test Plan: mcrouter unit tests

Reviewed By: andrii@fb.com

Subscribers: alikhtarov, folly-diffs@, yfeldblum, chalfant

FB internal diff: D1965097

Signature: t1:1965097:1428103637:341ff705c9d11c730197828256ec0b790ee7afaa

folly/experimental/fibers/Fiber.cpp

@@ -38,13 +38,6 @@ pid_t localThreadId() {
   return threadId;
 }
 
-static void fillMagic(const FContext& context) {
-  uint64_t* begin = static_cast<uint64_t*>(context.stackLimit());
-  uint64_t* end = static_cast<uint64_t*>(context.stackBase());
-
-  std::fill(begin, end, kMagic8Bytes);
-}
-
 /* Size of the region from p + nBytes down to the last non-magic value */
 static size_t nonMagicInBytes(const FContext& context) {
   uint64_t* begin = static_cast<uint64_t*>(context.stackLimit());
@@ -82,9 +75,24 @@ Fiber::Fiber(FiberManager& fiberManager) :
   auto limit = fiberManager_.stackAllocator_.allocate(size);
 
   fcontext_ = makeContext(limit, size, &Fiber::fiberFuncHelper);
+}
+
+void Fiber::init(bool recordStackUsed) {
+  recordStackUsed_ = recordStackUsed;
+  if (UNLIKELY(recordStackUsed_ && !stackFilledWithMagic_)) {
+    auto limit = fcontext_.stackLimit();
+    auto base = fcontext_.stackBase();
+
+    std::fill(static_cast<uint64_t*>(limit),
+              static_cast<uint64_t*>(base),
+              kMagic8Bytes);
+
+    // newer versions of boost allocate context on fiber stack,
+    // need to create a new one
+    auto size = fiberManager_.options_.stackSize;
+    fcontext_ = makeContext(limit, size, &Fiber::fiberFuncHelper);
 
-  if (UNLIKELY(fiberManager_.options_.debugRecordStackUsed)) {
-    fillMagic(fcontext_);
+    stackFilledWithMagic_ = true;
   }
 }
 
@@ -96,12 +104,12 @@ Fiber::~Fiber() {
 
 void Fiber::recordStackPosition() {
   int stackDummy;
-  fiberManager_.stackHighWatermark_ =
-    std::max(fiberManager_.stackHighWatermark_,
-             static_cast<size_t>(
+  auto currentPosition = static_cast<size_t>(
      static_cast<unsigned char*>(fcontext_.stackBase()) -
-               static_cast<unsigned char*>(
-                 static_cast<void*>(&stackDummy))));
+     static_cast<unsigned char*>(static_cast<void*>(&stackDummy)));
+  fiberManager_.stackHighWatermark_ =
+    std::max(fiberManager_.stackHighWatermark_, currentPosition);
+  VLOG(4) << "Stack usage: " << currentPosition;
 }
 
 void Fiber::fiberFuncHelper(intptr_t fiber) {
@@ -141,10 +149,13 @@ void Fiber::fiberFunc() {
                                        "running Fiber func_/resultFunc_");
     }
 
-    if (UNLIKELY(fiberManager_.options_.debugRecordStackUsed)) {
+    if (UNLIKELY(recordStackUsed_)) {
       fiberManager_.stackHighWatermark_ =
         std::max(fiberManager_.stackHighWatermark_,
                  nonMagicInBytes(fcontext_));
+      VLOG(3) << "Max stack usage: " << fiberManager_.stackHighWatermark_;
+      CHECK(fiberManager_.stackHighWatermark_ <
+              fiberManager_.options_.stackSize - 64) << "Fiber stack overflow";
     }
 
     state_ = INVALID;

folly/experimental/fibers/Fiber.h

@@ -68,6 +68,7 @@ class Fiber {
   friend class FiberManager;
 
   explicit Fiber(FiberManager& fiberManager);
+  void init(bool recordStackUsed);
 
   template <typename F>
   void setFunction(F&& func);
@@ -98,6 +99,8 @@ class Fiber {
   FContext fcontext_;           /**< current task execution context */
   intptr_t data_;               /**< Used to keep some data with the Fiber */
   std::function<void()> func_;  /**< task function */
+  bool recordStackUsed_{false};
+  bool stackFilledWithMagic_{false};
 
   /**
    * Points to next fiber in remote ready list

folly/experimental/fibers/FiberManager-inl.h

@@ -377,7 +377,7 @@ inline FiberManager* FiberManager::getFiberManagerUnsafe() {
   return currentFiberManager_;
 }
 
-inline bool FiberManager::hasActiveFiber() {
+inline bool FiberManager::hasActiveFiber() const {
   return activeFiber_ != nullptr;
 }
 

folly/experimental/fibers/FiberManager.cpp

@@ -93,8 +93,12 @@ Fiber* FiberManager::getFiber() {
     assert(fibersPoolSize_ > 0);
     --fibersPoolSize_;
   }
-  ++fibersActive_;
   assert(fiber);
+  ++fibersActive_;
+  ++fiberId_;
+  bool recordStack = (options_.recordStackEvery != 0) &&
+                     (fiberId_ % options_.recordStackEvery == 0);
+  fiber->init(recordStack);
   return fiber;
 }
 

folly/experimental/fibers/FiberManager.h

@@ -74,8 +74,9 @@ class FiberManager {
      * This is fairly expensive: we fill each newly allocated stack
      * with some known value and find the boundary of unused stack
      * with linear search every time we surrender the stack back to fibersPool.
+     * 0 disables stack recording.
      */
-    bool debugRecordStackUsed{false};
+    size_t recordStackEvery{0};
 
     /**
      * Keep at most this many free fibers in the pool.
@@ -194,7 +195,7 @@ class FiberManager {
   /**
    * return     true if running activeFiber_ is not nullptr.
    */
-  bool hasActiveFiber();
+  bool hasActiveFiber() const;
 
   /**
    * @return What was the most observed fiber stack usage (in bytes).
@@ -239,6 +240,7 @@ class FiberManager {
   size_t fibersAllocated_{0};   /**< total number of fibers allocated */
   size_t fibersPoolSize_{0};    /**< total number of fibers in the free pool */
   size_t fibersActive_{0};      /**< number of running or blocked fibers */
+  size_t fiberId_{0};           /**< id of last fiber used */
 
   FContext::ContextStruct mainContext_;  /**< stores loop function context */