✅ annotate non-reproducible tests #2324

README.md

@@ -1629,4 +1629,6 @@ Note that during the `ctest` stage, several JSON test files are downloaded from
 
 In case you have downloaded the library rather than checked out the code via Git, test `cmake_fetch_content_configure`. Please execute `ctest -LE git_required` to skip these tests. See [issue #2189](https://github.com/nlohmann/json/issues/2189) for more information.
 
+Some tests make the test suite not reproducible. Please execute `ctest -LE not_reproducible` to skip these tests. See [issue #2324](https://github.com/nlohmann/json/issues/2324) for more information.
+
 As Intel compilers use unsafe floating point optimization by default, the unit tests may fail. Use flag [`/fp:precise`](https://software.intel.com/content/www/us/en/develop/documentation/cpp-compiler-developer-guide-and-reference/top/compiler-reference/compiler-options/compiler-option-details/floating-point-options/fp-model-fp.html) then.

test/cmake_add_subdirectory/CMakeLists.txt

@@ -10,7 +10,9 @@ add_test(NAME cmake_add_subdirectory_build
 )
 set_tests_properties(cmake_add_subdirectory_configure PROPERTIES
   FIXTURES_SETUP cmake_add_subdirectory
+  LABELS not_reproducible
 )
 set_tests_properties(cmake_add_subdirectory_build PROPERTIES
   FIXTURES_REQUIRED cmake_add_subdirectory
+  LABELS not_reproducible
 )

test/cmake_fetch_content/CMakeLists.txt

@@ -12,9 +12,11 @@ if (${CMAKE_VERSION} VERSION_GREATER "3.11.0")
   set_tests_properties(cmake_fetch_content_configure PROPERTIES
     FIXTURES_SETUP cmake_fetch_content
     LABELS git_required
+    LABELS not_reproducible
   )
   set_tests_properties(cmake_fetch_content_build PROPERTIES
     FIXTURES_REQUIRED cmake_fetch_content
     LABELS git_required
+    LABELS not_reproducible
   )
 endif()

test/cmake_import/CMakeLists.txt

@@ -11,7 +11,9 @@ add_test(NAME cmake_import_build
 )
 set_tests_properties(cmake_import_configure PROPERTIES
   FIXTURES_SETUP cmake_import
+  LABELS not_reproducible
 )
 set_tests_properties(cmake_import_build PROPERTIES
   FIXTURES_REQUIRED cmake_import
+  LABELS not_reproducible
 )

test/cmake_import_minver/CMakeLists.txt

@@ -11,7 +11,9 @@ add_test(NAME cmake_import_minver_build
 )
 set_tests_properties(cmake_import_minver_configure PROPERTIES
   FIXTURES_SETUP cmake_import_minver
+  LABELS not_reproducible
 )
 set_tests_properties(cmake_import_minver_build PROPERTIES
   FIXTURES_REQUIRED cmake_import_minver
+  LABELS not_reproducible
 )

test/cmake_target_include_directories/CMakeLists.txt

@@ -10,7 +10,9 @@ add_test(NAME cmake_target_include_directories_build
 )
 set_tests_properties(cmake_target_include_directories_configure PROPERTIES
     FIXTURES_SETUP cmake_target_include_directories
+    LABELS not_reproducible
 )
 set_tests_properties(cmake_target_include_directories_build PROPERTIES
     FIXTURES_REQUIRED cmake_target_include_directories
+    LABELS not_reproducible
 )