Expand DSched interface for managing auxiliary functions

Summary:
Changed DSched interface for managing auxiliary functions to allow separate auxiliary functions for single actions (applicable to the next shared access by a specific thread) and repeating actions (applicable to all subsequent shared accesses).

[Note: I have a dependent diff that depends on both this diff and the diff for dynamic MPMCQueue (/D3462592). I don't think I can submit a diff that depends on multiple diffs that haven't landed yet. So, I'll wait until this one lands.]

Reviewed By: djwatson

Differential Revision: D3792669

fbshipit-source-id: 52654fffda2dc905b19ff91f4459f15da11f7735

folly/test/DeterministicSchedule.cpp

@@ -33,7 +33,8 @@ namespace test {
 FOLLY_TLS sem_t* DeterministicSchedule::tls_sem;
 FOLLY_TLS DeterministicSchedule* DeterministicSchedule::tls_sched;
 FOLLY_TLS unsigned DeterministicSchedule::tls_threadId;
-FOLLY_TLS std::function<void(uint64_t, bool)>* DeterministicSchedule::tls_aux;
+thread_local AuxAct DeterministicSchedule::tls_aux_act;
+AuxChk DeterministicSchedule::aux_chk;
 
 // access is protected by futexLock
 static std::unordered_map<detail::Futex<DeterministicAtomic>*,
@@ -46,7 +47,7 @@ DeterministicSchedule::DeterministicSchedule(
     : scheduler_(scheduler), nextThreadId_(1), step_(0) {
   assert(tls_sem == nullptr);
   assert(tls_sched == nullptr);
-  assert(tls_aux == nullptr);
+  assert(tls_aux_act == nullptr);
 
   tls_sem = new sem_t;
   sem_init(tls_sem, 0, 1);
@@ -171,8 +172,16 @@ int DeterministicSchedule::getcpu(unsigned* cpu,
   return 0;
 }
 
-void DeterministicSchedule::setAux(std::function<void(uint64_t, bool)>& aux) {
-  tls_aux = &aux;
+void DeterministicSchedule::setAuxAct(AuxAct& aux) {
+  tls_aux_act = aux;
+}
+
+void DeterministicSchedule::setAuxChk(AuxChk& aux) {
+  aux_chk = aux;
+}
+
+void DeterministicSchedule::clearAuxChk() {
+  aux_chk = nullptr;
 }
 
 sem_t* DeterministicSchedule::beforeThreadCreate() {
@@ -212,6 +221,7 @@ void DeterministicSchedule::beforeThreadExit() {
   delete tls_sem;
   tls_sem = nullptr;
   tls_sched = nullptr;
+  tls_aux_act = nullptr;
 }
 
 void DeterministicSchedule::join(std::thread& child) {
@@ -232,12 +242,13 @@ void DeterministicSchedule::join(std::thread& child) {
 
 void DeterministicSchedule::callAux(bool success) {
   ++step_;
-  auto aux = tls_aux;
-  if (!aux) {
-    return;
+  if (tls_aux_act) {
+    tls_aux_act(success);
+    tls_aux_act = nullptr;
+  }
+  if (aux_chk) {
+    aux_chk(step_);
   }
-  (*aux)(step_, success);
-  tls_aux = nullptr;
 }
 
 void DeterministicSchedule::post(sem_t* sem) {

folly/test/DeterministicSchedule.h

@@ -51,6 +51,10 @@ namespace test {
     }                                                           \
   } while (false)
 
+/* signatures of user-defined auxiliary functions */
+using AuxAct = std::function<void(bool)>;
+using AuxChk = std::function<void(uint64_t)>;
+
 /**
  * DeterministicSchedule coordinates the inter-thread communication of a
  * set of threads under test, so that despite concurrency the execution is
@@ -168,18 +172,27 @@ class DeterministicSchedule : boost::noncopyable {
   static int getcpu(unsigned* cpu, unsigned* node, void* unused);
 
   /** Sets up a thread-specific function for call immediately after
-   *  the next shared access for managing auxiliary data and checking
-   *  global invariants. The parameters of the function are: a
-   *  uint64_t that indicates the step number (i.e., the number of
-   *  shared accesses so far), and a bool that indicates the success
-   *  of the shared access (if it is conditional, true otherwise). */
-  static void setAux(std::function<void(uint64_t, bool)>& aux);
+   *  the next shared access by the thread for managing auxiliary
+   *  data. The function takes a bool parameter that indicates the
+   *  success of the shared access (if it is conditional, true
+   *  otherwise). The function is cleared after one use. */
+  static void setAuxAct(AuxAct& aux);
+
+  /** Sets up a function to be called after every subsequent shared
+   *  access (until clearAuxChk() is called) for checking global
+   *  invariants and logging. The function takes a uint64_t parameter
+   *  that indicates the number of shared accesses so far. */
+  static void setAuxChk(AuxChk& aux);
+
+  /** Clears the function set by setAuxChk */
+  static void clearAuxChk();
 
  private:
   static FOLLY_TLS sem_t* tls_sem;
   static FOLLY_TLS DeterministicSchedule* tls_sched;
   static FOLLY_TLS unsigned tls_threadId;
-  static FOLLY_TLS std::function<void(uint64_t, bool)>* tls_aux;
+  static thread_local AuxAct tls_aux_act;
+  static AuxChk aux_chk;
 
   std::function<int(int)> scheduler_;
   std::vector<sem_t*> sems_;

folly/test/DeterministicScheduleTest.cpp

@@ -129,7 +129,6 @@ TEST(DeterministicSchedule, buggyAdd) {
 ///   8. Define TEST using anotated shared data, aux data, and aux functions
 
 using DSched = DeterministicSchedule;
-using AuxFn = std::function<void(uint64_t, bool)>;
 
 /** forward declaration of annotated shared class */
 class AnnotatedAtomicCounter;
@@ -180,17 +179,17 @@ class AnnotatedAtomicCounter {
  public:
   explicit AnnotatedAtomicCounter(int val) : shared_(val) {}
 
-  void inc(AuxFn& auxfn) {
-    DSched::setAux(auxfn);
+  void inc(AuxAct& auxfn) {
+    DSched::setAuxAct(auxfn);
     /* calls the fine-grained original */
     shared_.inc();
   }
 
-  void inc_bug(AuxFn auxfn) {
+  void inc_bug(AuxAct auxfn) {
     /* duplicates the steps of the multi-access original in order to
      * annotate the second access */
     int newval = shared_.counter_.load() + 1;
-    DSched::setAux(auxfn);
+    DSched::setAuxAct(auxfn);
     shared_.counter_.store(newval);
   }
 
@@ -205,7 +204,7 @@ class AnnotatedAtomicCounter {
 using Annotated = AnnotatedAtomicCounter;
 
 /** aux log & check function */
-void auxCheck(int tid, uint64_t step, Annotated& annotated, AuxData& auxdata) {
+void auxCheck(int tid, Annotated& annotated, AuxData& auxdata) {
   /* read shared data */
   int val = annotated.load_direct();
   /* read auxiliary data */
@@ -214,21 +213,20 @@ void auxCheck(int tid, uint64_t step, Annotated& annotated, AuxData& auxdata) {
     sum += v;
   }
   /* log state */
-  VLOG(2) << "Step " << step << " -- tid " << tid
-          << " -- shared counter= " << val << " -- sum increments= " << sum;
+  VLOG(2) << "tid " << tid << " -- shared counter= " << val
+          << " -- sum increments= " << sum;
   /* check invariant */
   if (val != sum) {
-    LOG(ERROR) << "Failed after step " << step;
     LOG(ERROR) << "counter=(" << val << ") expected(" << sum << ")";
     CHECK(false);
   }
 }
 
 /** function generator(s) */
-AuxFn auxAfterInc(int tid, Annotated& annotated, AuxData& auxdata) {
-  return [&annotated, &auxdata, tid](uint64_t step, bool success) {
+AuxAct auxAfterInc(int tid, Annotated& annotated, AuxData& auxdata) {
+  return [&annotated, &auxdata, tid](bool success) {
     auxUpdateAfterInc(tid, auxdata, success);
-    auxCheck(tid, step, annotated, auxdata);
+    auxCheck(tid, annotated, auxdata);
   };
 }
 
@@ -252,7 +250,7 @@ TEST(DSchedCustom, atomic_add) {
   std::vector<std::thread> threads(nthr);
   for (int tid = 0; tid < nthr; ++tid) {
     threads[tid] = DSched::thread([&, tid]() {
-      AuxFn auxfn = auxAfterInc(tid, annotated, auxdata);
+      AuxAct auxfn = auxAfterInc(tid, annotated, auxdata);
       for (int i = 0; i < niter; ++i) {
         if (bug && (tid == 0) && (i % 10 == 0)) {
           annotated.inc_bug(auxfn);