logging: add LoggerDB::updateConfig() and resetConfig()

Summary: Add methods for applying config changes from a LogConfig object to the LoggerDB.

Reviewed By: bolinfest

Differential Revision: D6200564

fbshipit-source-id: a25eb99e84b2885bf6853e2222db0d7432a6c37b

CMakeLists.txt

@@ -289,6 +289,7 @@ if (BUILD_TESTS)
     ${FOLLY_DIR}/test/SingletonTestStructs.cpp
     ${FOLLY_DIR}/test/SocketAddressTestHelper.cpp
     ${FOLLY_DIR}/test/SocketAddressTestHelper.h
+    ${FOLLY_DIR}/experimental/logging/test/TestLogHandler.cpp
     ${FOLLY_DIR}/experimental/logging/test/TestLogHandler.h
     ${FOLLY_DIR}/futures/test/TestExecutor.cpp
     ${FOLLY_DIR}/futures/test/TestExecutor.h
@@ -386,6 +387,7 @@ if (BUILD_TESTS)
     DIRECTORY experimental/logging/test/
       TEST async_file_writer_test SOURCES AsyncFileWriterTest.cpp
       TEST config_parser_test SOURCES ConfigParserTest.cpp
+      TEST config_update_test SOURCES ConfigUpdateTest.cpp
       TEST file_handler_factory_test SOURCES FileHandlerFactoryTest.cpp
       TEST glog_formatter_test SOURCES GlogFormatterTest.cpp
       TEST immediate_file_writer_test SOURCES ImmediateFileWriterTest.cpp

folly/experimental/logging/LogCategory.cpp

@@ -21,6 +21,7 @@
 #include <folly/ConstexprMath.h>
 #include <folly/ExceptionString.h>
 #include <folly/FileUtil.h>
+#include <folly/MapUtil.h>
 #include <folly/experimental/logging/LogHandler.h>
 #include <folly/experimental/logging/LogMessage.h>
 #include <folly/experimental/logging/LogName.h>
@@ -144,6 +145,23 @@ std::vector<std::shared_ptr<LogHandler>> LogCategory::getHandlers() const {
   return *(handlers_.rlock());
 }
 
+void LogCategory::replaceHandlers(
+    std::vector<std::shared_ptr<LogHandler>> handlers) {
+  return handlers_.wlock()->swap(handlers);
+}
+
+void LogCategory::updateHandlers(const std::unordered_map<
+                                 std::shared_ptr<LogHandler>,
+                                 std::shared_ptr<LogHandler>>& handlerMap) {
+  auto handlers = handlers_.wlock();
+  for (auto& entry : *handlers) {
+    auto* ptr = get_ptr(handlerMap, entry);
+    if (ptr) {
+      entry = *ptr;
+    }
+  }
+}
+
 void LogCategory::setLevel(LogLevel level, bool inherit) {
   // We have to set the level through LoggerDB, since we require holding
   // the LoggerDB lock to iterate through our children in case our effective

folly/experimental/logging/LogCategory.h

@@ -164,6 +164,26 @@ class LogCategory {
    */
   std::vector<std::shared_ptr<LogHandler>> getHandlers() const;
 
+  /**
+   * Replace the list of LogHandlers with a completely new list.
+   */
+  void replaceHandlers(std::vector<std::shared_ptr<LogHandler>> handlers);
+
+  /**
+   * Update the LogHandlers attached to this LogCategory by replacing
+   * currently attached handlers with new LogHandler objects.
+   *
+   * The handlerMap argument is a map of (old_handler -> new_handler)
+   * If any of the LogHandlers currently attached to this category are found in
+   * the handlerMap, replace them with the new handler indicated in the map.
+   *
+   * This is used when the LogHandler configuration is changed requiring one or
+   * more LogHandler objects to be replaced with new ones.
+   */
+  void updateHandlers(const std::unordered_map<
+                      std::shared_ptr<LogHandler>,
+                      std::shared_ptr<LogHandler>>& handlerMap);
+
   /* Internal methods for use by other parts of the logging library code */
 
   /**

folly/experimental/logging/LoggerDB.cpp

@@ -208,6 +208,229 @@ LogConfig LoggerDB::getConfig() const {
   return LogConfig{std::move(handlerConfigs), std::move(categoryConfigs)};
 }
 
+/**
+ * Process handler config information when starting a config update operation.
+ */
+void LoggerDB::startConfigUpdate(
+    const Synchronized<HandlerInfo>::LockedPtr& handlerInfo,
+    const LogConfig& config,
+    NewHandlerMap* handlers,
+    OldToNewHandlerMap* oldToNewHandlerMap) {
+  // Get a map of all currently existing LogHandlers.
+  // This resolves weak_ptrs to shared_ptrs, and ignores expired weak_ptrs.
+  // This prevents any of these LogHandler pointers from expiring during the
+  // config update.
+  for (const auto& entry : handlerInfo->handlers) {
+    auto handler = entry.second.lock();
+    if (handler) {
+      handlers->emplace(entry.first, std::move(handler));
+    }
+  }
+
+  // Create all of the new LogHandlers needed from this configuration
+  for (const auto& entry : config.getHandlerConfigs()) {
+    // Look up the LogHandlerFactory
+    auto factoryIter = handlerInfo->factories.find(entry.second.type);
+    if (factoryIter == handlerInfo->factories.end()) {
+      throw std::invalid_argument(to<std::string>(
+          "unknown log handler type \"", entry.second.type, "\""));
+    }
+
+    // Check to see if there is an existing LogHandler with this name
+    std::shared_ptr<LogHandler> oldHandler;
+    auto iter = handlers->find(entry.first);
+    if (iter != handlers->end()) {
+      oldHandler = iter->second;
+    }
+
+    // Create the new log handler
+    const auto& factory = factoryIter->second;
+    std::shared_ptr<LogHandler> handler;
+    if (oldHandler) {
+      handler = factory->updateHandler(oldHandler, entry.second.options);
+      if (handler != oldHandler) {
+        oldToNewHandlerMap->emplace(oldHandler, handler);
+      }
+    } else {
+      handler = factory->createHandler(entry.second.options);
+    }
+    handlerInfo->handlers[entry.first] = handler;
+    (*handlers)[entry.first] = handler;
+  }
+
+  // Before we start making any LogCategory changes, confirm that all handlers
+  // named in the category configs are known handlers.
+  for (const auto& entry : config.getCategoryConfigs()) {
+    if (!entry.second.handlers.hasValue()) {
+      continue;
+    }
+    for (const auto& handlerName : entry.second.handlers.value()) {
+      auto iter = handlers->find(handlerName);
+      if (iter == handlers->end()) {
+        throw std::invalid_argument(to<std::string>(
+            "unknown log handler \"",
+            handlerName,
+            "\" configured for log category \"",
+            entry.first,
+            "\""));
+      }
+    }
+  }
+}
+
+/**
+ * Update handlerInfo_ at the end of a config update operation.
+ */
+void LoggerDB::finishConfigUpdate(
+    const Synchronized<HandlerInfo>::LockedPtr& handlerInfo,
+    NewHandlerMap* handlers,
+    OldToNewHandlerMap* oldToNewHandlerMap) {
+  // Build a new map to replace handlerInfo->handlers
+  // This will contain only the LogHandlers that are still in use by the
+  // current LogCategory settings.
+  std::unordered_map<std::string, std::weak_ptr<LogHandler>> newHandlerMap;
+  for (const auto& entry : *handlers) {
+    newHandlerMap.emplace(entry.first, entry.second);
+  }
+  // Drop all of our shared_ptr references to LogHandler objects,
+  // and then remove entries in newHandlerMap that are unreferenced.
+  handlers->clear();
+  oldToNewHandlerMap->clear();
+  handlerInfo->handlers.clear();
+  for (auto iter = newHandlerMap.begin(); iter != newHandlerMap.end(); /**/) {
+    if (iter->second.expired()) {
+      iter = newHandlerMap.erase(iter);
+    } else {
+      ++iter;
+    }
+  }
+  handlerInfo->handlers.swap(newHandlerMap);
+}
+
+std::vector<std::shared_ptr<LogHandler>> LoggerDB::buildCategoryHandlerList(
+    const NewHandlerMap& handlerMap,
+    StringPiece categoryName,
+    const std::vector<std::string>& categoryHandlerNames) {
+  std::vector<std::shared_ptr<LogHandler>> catHandlers;
+  for (const auto& handlerName : categoryHandlerNames) {
+    auto iter = handlerMap.find(handlerName);
+    if (iter == handlerMap.end()) {
+      // This really shouldn't be possible; the checks in startConfigUpdate()
+      // should have already bailed out if there was an unknown handler.
+      throw std::invalid_argument(to<std::string>(
+          "bug: unknown log handler \"",
+          handlerName,
+          "\" configured for log category \"",
+          categoryName,
+          "\""));
+    }
+    catHandlers.push_back(iter->second);
+  }
+
+  return catHandlers;
+}
+
+void LoggerDB::updateConfig(const LogConfig& config) {
+  // Grab the handlerInfo_ lock.
+  // We hold it in write mode for the entire config update operation.  This
+  // ensures that only a single config update operation ever runs at once.
+  auto handlerInfo = handlerInfo_.wlock();
+
+  NewHandlerMap handlers;
+  OldToNewHandlerMap oldToNewHandlerMap;
+  startConfigUpdate(handlerInfo, config, &handlers, &oldToNewHandlerMap);
+
+  // If an existing LogHandler was replaced with a new one,
+  // walk all current LogCategories and replace this handler.
+  if (!oldToNewHandlerMap.empty()) {
+    auto loggerMap = loggersByName_.rlock();
+    for (const auto& entry : *loggerMap) {
+      entry.second->updateHandlers(oldToNewHandlerMap);
+    }
+  }
+
+  // Update log levels and handlers mentioned in the config update
+  auto loggersByName = loggersByName_.wlock();
+  for (const auto& entry : config.getCategoryConfigs()) {
+    LogCategory* category =
+        getOrCreateCategoryLocked(*loggersByName, entry.first);
+
+    // Update the log handlers
+    if (entry.second.handlers.hasValue()) {
+      auto catHandlers = buildCategoryHandlerList(
+          handlers, entry.first, entry.second.handlers.value());
+      category->replaceHandlers(std::move(catHandlers));
+    }
+
+    // Update the level settings
+    category->setLevelLocked(
+        entry.second.level, entry.second.inheritParentLevel);
+  }
+
+  finishConfigUpdate(handlerInfo, &handlers, &oldToNewHandlerMap);
+}
+
+void LoggerDB::resetConfig(const LogConfig& config) {
+  // Grab the handlerInfo_ lock.
+  // We hold it in write mode for the entire config update operation.  This
+  // ensures that only a single config update operation ever runs at once.
+  auto handlerInfo = handlerInfo_.wlock();
+
+  NewHandlerMap handlers;
+  OldToNewHandlerMap oldToNewHandlerMap;
+  startConfigUpdate(handlerInfo, config, &handlers, &oldToNewHandlerMap);
+
+  // Make sure all log categories mentioned in the new config exist.
+  // This ensures that we will cover them in our walk below.
+  LogCategory* rootCategory;
+  {
+    auto loggersByName = loggersByName_.wlock();
+    rootCategory = getOrCreateCategoryLocked(*loggersByName, "");
+    for (const auto& entry : config.getCategoryConfigs()) {
+      getOrCreateCategoryLocked(*loggersByName, entry.first);
+    }
+  }
+
+  {
+    // Update all log categories
+    auto loggersByName = loggersByName_.rlock();
+    for (const auto& entry : *loggersByName) {
+      auto* category = entry.second.get();
+
+      auto configIter = config.getCategoryConfigs().find(category->getName());
+      if (configIter == config.getCategoryConfigs().end()) {
+        // This category is not listed in the config settings.
+        // Reset it to the default settings.
+        category->clearHandlers();
+
+        if (category == rootCategory) {
+          category->setLevelLocked(LogLevel::ERR, false);
+        } else {
+          category->setLevelLocked(LogLevel::MAX_LEVEL, true);
+        }
+        continue;
+      }
+
+      const auto& catConfig = configIter->second;
+
+      // Update the category log level
+      category->setLevelLocked(catConfig.level, catConfig.inheritParentLevel);
+
+      // Update the category handlers list.
+      // If the handler list is not set in the config, clear out any existing
+      // handlers rather than leaving it as-is.
+      std::vector<std::shared_ptr<LogHandler>> catHandlers;
+      if (catConfig.handlers.hasValue()) {
+        catHandlers = buildCategoryHandlerList(
+            handlers, entry.first, catConfig.handlers.value());
+      }
+      category->replaceHandlers(std::move(catHandlers));
+    }
+  }
+
+  finishConfigUpdate(handlerInfo, &handlers, &oldToNewHandlerMap);
+}
+
 LogCategory* LoggerDB::getOrCreateCategoryLocked(
     LoggerNameMap& loggersByName,
     StringPiece name) {

folly/experimental/logging/LoggerDB.h

@@ -88,6 +88,29 @@ class LoggerDB {
    */
   LogConfig getConfig() const;
 
+  /**
+   * Update the current LoggerDB state with the specified LogConfig settings.
+   *
+   * Log categories and handlers listed in the LogConfig object will be updated
+   * to the new state listed in the LogConfig.  Settings on categories and
+   * handlers not listed in the config will be left as-is.
+   */
+  void updateConfig(const LogConfig& config);
+
+  /**
+   * Reset the current LoggerDB state to the specified LogConfig settings.
+   *
+   * All LogCategories not mentioned in the new LogConfig will have all
+   * currently configured log handlers removed and their log level set to its
+   * default state.  For the root category the default log level is ERR; for
+   * all other categories the default level is MAX_LEVEL with log level
+   * inheritance enabled.
+   *
+   * LogCategories listed in the new config but without LogHandler information
+   * defined will have all existing handlers removed.
+   */
+  void resetConfig(const LogConfig& config);
+
   /**
    * Apply a configuration string specifying a series a log levels.
    *
@@ -237,6 +260,24 @@ class LoggerDB {
       folly::StringPiece name,
       LogCategory* parent);
 
+  using NewHandlerMap =
+      std::unordered_map<std::string, std::shared_ptr<LogHandler>>;
+  using OldToNewHandlerMap = std::
+      unordered_map<std::shared_ptr<LogHandler>, std::shared_ptr<LogHandler>>;
+  void startConfigUpdate(
+      const Synchronized<HandlerInfo>::LockedPtr& handlerInfo,
+      const LogConfig& config,
+      NewHandlerMap* handlers,
+      OldToNewHandlerMap* oldToNewHandlerMap);
+  void finishConfigUpdate(
+      const Synchronized<HandlerInfo>::LockedPtr& handlerInfo,
+      NewHandlerMap* handlers,
+      OldToNewHandlerMap* oldToNewHandlerMap);
+  std::vector<std::shared_ptr<LogHandler>> buildCategoryHandlerList(
+      const NewHandlerMap& handlerMap,
+      StringPiece categoryName,
+      const std::vector<std::string>& categoryHandlerNames);
+
   static void internalWarningImpl(
       folly::StringPiece filename,
       int lineNumber,
@@ -256,6 +297,9 @@ class LoggerDB {
 
   /**
    * The LogHandlers and LogHandlerFactories.
+   *
+   * For lock ordering purposes, if you need to acquire both the loggersByName_
+   * and handlerInfo_ locks, the handlerInfo_ lock must be acquired first.
    */
   folly::Synchronized<HandlerInfo> handlerInfo_;
 

folly/experimental/logging/test/TestLogHandler.cpp

+/*
+ * Copyright 2004-present Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include <folly/experimental/logging/test/TestLogHandler.h>
+
+#include <folly/MapUtil.h>
+
+namespace folly {
+
+std::shared_ptr<LogHandler> TestLogHandlerFactory::createHandler(
+    const Options& options) {
+  return std::make_shared<TestLogHandler>(LogHandlerConfig{type_, options});
+}
+
+std::shared_ptr<LogHandler> TestLogHandlerFactory::updateHandler(
+    const std::shared_ptr<LogHandler>& existingHandler,
+    const Options& options) {
+  // Only re-use an existing handler in-place if it is a TestLogHandler
+  // and if the new options contain reuse_handler
+  auto existing = std::dynamic_pointer_cast<TestLogHandler>(existingHandler);
+  if (!existing || !get_ptr(options, "reuse_handler")) {
+    return createHandler(options);
+  }
+
+  existing->setOptions(options);
+  return existing;
+}
+
+} // namespace folly

folly/experimental/logging/test/TestLogHandler.h

@@ -22,6 +22,7 @@
 
 #include <folly/experimental/logging/LogHandler.h>
 #include <folly/experimental/logging/LogHandlerConfig.h>
+#include <folly/experimental/logging/LogHandlerFactory.h>
 #include <folly/experimental/logging/LogMessage.h>
 
 namespace folly {
@@ -34,6 +35,8 @@ namespace folly {
  */
 class TestLogHandler : public LogHandler {
  public:
+  using Options = LogHandlerConfig::Options;
+
   TestLogHandler() : config_{"test"} {}
   explicit TestLogHandler(LogHandlerConfig config)
       : config_{std::move(config)} {}
@@ -60,10 +63,36 @@ class TestLogHandler : public LogHandler {
     return config_;
   }
 
- private:
+  void setOptions(const Options& options) {
+    config_.options = options;
+  }
+
+ protected:
   std::vector<std::pair<LogMessage, const LogCategory*>> messages_;
   uint64_t flushCount_{0};
   std::map<std::string, std::string> options_;
   LogHandlerConfig config_;
 };
+
+/**
+ * A LogHandlerFactory to create TestLogHandler objects.
+ */
+class TestLogHandlerFactory : public LogHandlerFactory {
+ public:
+  explicit TestLogHandlerFactory(StringPiece type) : type_{type.str()} {}
+
+  StringPiece getType() const override {
+    return type_;
+  }
+
+  std::shared_ptr<LogHandler> createHandler(const Options& options) override;
+
+  std::shared_ptr<LogHandler> updateHandler(
+      const std::shared_ptr<LogHandler>& existingHandler,
+      const Options& options) override;
+
+ private:
+  std::string type_;
+};
+
 } // namespace folly