logging: add LogFormatter and LogWriter interfaces

Summary:
This simplifies the LogHandler interface to a single generic `handleMessage()`,
and adds a `StandardLogHandler` implementation that defers to separate
`LogFormatter` and `LogWriter` objects.

The `LogFormatter` class is responsible for serializing the `LogMessage` object
into a string, and `LogWriter` is responsible for then doing something with
the serialized string.

This will make it possible in the future to have separate `LogWriter`
implementations that all share the same log formatting code.  For example, this
will allow separate `LogWriter` implementations for performing file I/O
immediately versus performing I/O asynchronously in a separate thread.

Reviewed By: yfeldblum

Differential Revision: D5083103

fbshipit-source-id: e3f5ece25e260c825d49a5eb30e942973d6b68bf

CMakeLists.txt

@@ -327,6 +327,7 @@ if (BUILD_TESTS)
           LogMessageTest.cpp
           LogNameTest.cpp
           LogStreamTest.cpp
+          StandardLogHandlerTest.cpp
           XlogFile1.cpp
           XlogFile2.cpp
           XlogTest.cpp

folly/Makefile.am

@@ -122,14 +122,17 @@ nobase_follyinclude_HEADERS = \
 	experimental/JSONSchema.h \
 	experimental/LockFreeRingBuffer.h \
 	experimental/logging/LogCategory.h \
+	experimental/logging/LogFormatter.h \
+	experimental/logging/Logger.h \
+	experimental/logging/LoggerDB.h \
 	experimental/logging/LogHandler.h \
 	experimental/logging/LogLevel.h \
 	experimental/logging/LogMessage.h \
 	experimental/logging/LogName.h \
 	experimental/logging/LogStream.h \
 	experimental/logging/LogStreamProcessor.h \
-	experimental/logging/Logger.h \
-	experimental/logging/LoggerDB.h \
+	experimental/logging/LogWriter.h \
+	experimental/logging/StandardLogHandler.h \
 	experimental/logging/xlog.h \
 	experimental/NestedCommandLineApp.h \
 	experimental/observer/detail/Core.h \

folly/experimental/logging/LogCategory.cpp

@@ -69,7 +69,7 @@ void LogCategory::processMessage(const LogMessage& message) const {
 
   for (size_t n = 0; n < numHandlers; ++n) {
     try {
-      handlers[n]->log(message, this);
+      handlers[n]->handleMessage(message, this);
     } catch (const std::exception& ex) {
       // If a LogHandler throws an exception, complain about this fact on
       // stderr to avoid swallowing the error information completely.  We

folly/experimental/logging/LogFormatter.h

+/*
+ * Copyright 2004-present Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#include <string>
+
+namespace folly {
+
+class LogCategory;
+class LogMessage;
+
+/**
+ * LogFormatter defines the interface for serializing a LogMessage object
+ * into a buffer to be given to a LogWriter.
+ */
+class LogFormatter {
+ public:
+  virtual ~LogFormatter() {}
+
+  /**
+   * Serialze a LogMessage object.
+   *
+   * @param message The LogMessage object to serialze.
+   * @param handlerCategory The LogCategory that is currently handling this
+   *     message.  Note that this is likely different from the LogCategory
+   *     where the message was originally logged, which can be accessed as
+   *     message->getCategory()
+   */
+  virtual std::string formatMessage(
+      const LogMessage& message,
+      const LogCategory* handlerCategory) = 0;
+};
+}

folly/experimental/logging/LogHandler.h

@@ -42,33 +42,11 @@ class LogMessage;
  */
 class LogHandler {
  public:
-  LogHandler() = default;
   virtual ~LogHandler() = default;
 
   /**
-   * log() is called when a log message is processed by a LogCategory that this
-   * handler is attached to.
-   *
-   * log() performs a level check, and calls handleMessage() if it passes.
-   *
-   * @param message The LogMessage objet.
-   * @param handlerCategory  The LogCategory that invoked log().  This is the
-   *     category that this LogHandler is attached to.  Note that this may be
-   *     different than the category that this message was originally logged
-   *     at.  message->getCategory() returns the category of the log message.
-   */
-  void log(const LogMessage& message, const LogCategory* handlerCategory);
-
-  LogLevel getLevel() const {
-    return level_.load(std::memory_order_acquire);
-  }
-  void setLevel(LogLevel level) {
-    return level_.store(level, std::memory_order_release);
-  }
-
- protected:
-  /**
-   * handleMessage() is invoked to process a LogMessage.
+   * handleMessage() is called when a log message is processed by a LogCategory
+   * that this handler is attached to.
    *
    * This must be implemented by LogHandler subclasses.
    *
@@ -76,12 +54,16 @@ class LogHandler {
    * message.  LogMessage::getThreadID() contains the thread ID, but the
    * LogHandler can also include any other thread-local state they desire, and
    * this will always be data for the thread that originated the log message.
+   *
+   * @param message The LogMessage objet.
+   * @param handlerCategory  The LogCategory that invoked handleMessage().
+   *     This is the category that this LogHandler is attached to.  Note that
+   *     this may be different than the category that this message was
+   *     originally logged at.  message->getCategory() returns the category of
+   *     the log message.
    */
   virtual void handleMessage(
       const LogMessage& message,
       const LogCategory* handlerCategory) = 0;
-
- private:
-  std::atomic<LogLevel> level_{LogLevel::NONE};
 };
 }

folly/experimental/logging/LogWriter.h

+/*
+ * Copyright 2004-present Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#include <folly/Range.h>
+
+namespace folly {
+
+/**
+ * LogWriter defines the interface for processing a serialized log message.
+ */
+class LogWriter {
+ public:
+  virtual ~LogWriter() {}
+
+  /**
+   * Write a serialized log message.
+   */
+  virtual void writeMessage(folly::StringPiece buffer) = 0;
+
+  /**
+   * Write a serialized message.
+   *
+   * This version of writeMessage() accepts a std::string&&.
+   * The default implementation calls the StringPiece version of
+   * writeMessage(), but subclasses may override this implementation if
+   * desired.
+   */
+  virtual void writeMessage(std::string&& buffer) {
+    writeMessage(folly::StringPiece{buffer});
+  }
+};
+}

folly/experimental/logging/Makefile.am

@@ -6,12 +6,12 @@ libfollylogging_la_SOURCES = \
 	LogCategory.cpp \
 	Logger.cpp \
 	LoggerDB.cpp \
-	LogHandler.cpp \
 	LogLevel.cpp \
 	LogMessage.cpp \
 	LogName.cpp \
 	LogStream.cpp \
 	LogStreamProcessor.cpp \
+	StandardLogHandler.cpp \
 	xlog.cpp
 
 libfollylogging_la_LIBADD = $(top_builddir)/libfolly.la

folly/experimental/logging/LogHandler.cpp → folly/experimental/logging/StandardLogHandler.cpp

@@ -13,18 +13,27 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-#include <folly/experimental/logging/LogHandler.h>
+#include <folly/experimental/logging/StandardLogHandler.h>
 
+#include <folly/experimental/logging/LogFormatter.h>
 #include <folly/experimental/logging/LogMessage.h>
+#include <folly/experimental/logging/LogWriter.h>
 
 namespace folly {
 
-void LogHandler::log(
+StandardLogHandler::StandardLogHandler(
+    std::shared_ptr<LogFormatter> formatter,
+    std::shared_ptr<LogWriter> writer)
+    : formatter_{std::move(formatter)}, writer_{std::move(writer)} {}
+
+StandardLogHandler::~StandardLogHandler() {}
+
+void StandardLogHandler::handleMessage(
     const LogMessage& message,
     const LogCategory* handlerCategory) {
   if (message.getLevel() < getLevel()) {
     return;
   }
-  handleMessage(message, handlerCategory);
+  writer_->writeMessage(formatter_->formatMessage(message, handlerCategory));
 }
 }

folly/experimental/logging/StandardLogHandler.h

+/*
+ * Copyright 2004-present Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#include <memory>
+
+#include <folly/File.h>
+#include <folly/Range.h>
+#include <folly/experimental/logging/LogHandler.h>
+
+namespace folly {
+
+class LogFormatter;
+class LogWriter;
+
+/**
+ * StandardLogHandler is a LogHandler implementation that uses a LogFormatter
+ * class to serialize the LogMessage into a string, and then gives it to a
+ * LogWriter object.
+ *
+ * This basically is a simple glue class that helps chain together
+ * configurable LogFormatter and LogWriter objects.
+ *
+ * StandardLogHandler also supports ignoring messages less than a specific
+ * LogLevel.  By default it processes all messages.
+ */
+class StandardLogHandler : public LogHandler {
+ public:
+  StandardLogHandler(
+      std::shared_ptr<LogFormatter> formatter,
+      std::shared_ptr<LogWriter> writer);
+  ~StandardLogHandler();
+
+  /**
+   * Get the handler's current LogLevel.
+   *
+   * Messages less than this LogLevel will be ignored.  This defaults to
+   * LogLevel::NONE when the handler is constructed.
+   */
+  LogLevel getLevel() const {
+    return level_.load(std::memory_order_acquire);
+  }
+
+  /**
+   * Set the handler's current LogLevel.
+   *
+   * Messages less than this LogLevel will be ignored.
+   */
+  void setLevel(LogLevel level) {
+    return level_.store(level, std::memory_order_release);
+  }
+
+  void handleMessage(
+      const LogMessage& message,
+      const LogCategory* handlerCategory) override;
+
+ private:
+  std::atomic<LogLevel> level_{LogLevel::NONE};
+  std::shared_ptr<LogFormatter> formatter_;
+  std::shared_ptr<LogWriter> writer_;
+};
+}

folly/experimental/logging/test/StandardLogHandlerTest.cpp

+/*
+ * Copyright 2004-present Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include <folly/Conv.h>
+#include <folly/experimental/logging/LogCategory.h>
+#include <folly/experimental/logging/LogFormatter.h>
+#include <folly/experimental/logging/LogLevel.h>
+#include <folly/experimental/logging/LogMessage.h>
+#include <folly/experimental/logging/LogWriter.h>
+#include <folly/experimental/logging/LoggerDB.h>
+#include <folly/experimental/logging/StandardLogHandler.h>
+#include <folly/portability/GTest.h>
+
+using namespace folly;
+using std::make_shared;
+
+namespace {
+class TestLogFormatter : public LogFormatter {
+ public:
+  std::string formatMessage(
+      const LogMessage& message,
+      const LogCategory* handlerCategory) override {
+    return folly::to<std::string>(
+        logLevelToString(message.getLevel()),
+        "::",
+        message.getCategory()->getName(),
+        "::",
+        handlerCategory->getName(),
+        "::",
+        message.getFileName(),
+        "::",
+        message.getLineNumber(),
+        "::",
+        message.getMessage());
+  }
+};
+
+class TestLogWriter : public LogWriter {
+ public:
+  void writeMessage(folly::StringPiece buffer) override {
+    messages_.emplace_back(buffer.str());
+  }
+
+  std::vector<std::string>& getMessages() {
+    return messages_;
+  }
+  const std::vector<std::string>& getMessages() const {
+    return messages_;
+  }
+
+ private:
+  std::vector<std::string> messages_;
+};
+}
+
+TEST(StandardLogHandler, simple) {
+  auto writer = make_shared<TestLogWriter>();
+  StandardLogHandler handler(make_shared<TestLogFormatter>(), writer);
+
+  LoggerDB db{LoggerDB::TESTING};
+  auto logCategory = db.getCategory("log_cat");
+  auto handlerCategory = db.getCategory("handler_cat");
+
+  LogMessage msg{logCategory,
+                 LogLevel::DBG8,
+                 "src/test.cpp",
+                 1234,
+                 std::string{"hello world"}};
+  handler.handleMessage(msg, handlerCategory);
+  ASSERT_EQ(1, writer->getMessages().size());
+  EXPECT_EQ(
+      "LogLevel::DBG8::log_cat::handler_cat::src/test.cpp::1234::hello world",
+      writer->getMessages()[0]);
+}
+
+TEST(StandardLogHandler, levelCheck) {
+  auto writer = make_shared<TestLogWriter>();
+  StandardLogHandler handler(make_shared<TestLogFormatter>(), writer);
+
+  LoggerDB db{LoggerDB::TESTING};
+  auto logCategory = db.getCategory("log_cat");
+  auto handlerCategory = db.getCategory("handler_cat");
+
+  auto logMsg = [&](LogLevel level, folly::StringPiece message) {
+    LogMessage msg{logCategory, level, "src/test.cpp", 1234, message};
+    handler.handleMessage(msg, handlerCategory);
+  };
+
+  handler.setLevel(LogLevel::WARN);
+  logMsg(LogLevel::INFO, "info");
+  logMsg(LogLevel::WARN, "beware");
+  logMsg(LogLevel::ERR, "whoops");
+  logMsg(LogLevel::DBG1, "debug stuff");
+
+  auto& messages = writer->getMessages();
+  ASSERT_EQ(2, messages.size());
+  EXPECT_EQ(
+      "LogLevel::WARN::log_cat::handler_cat::src/test.cpp::1234::beware",
+      messages.at(0));
+  EXPECT_EQ(
+      "LogLevel::ERR::log_cat::handler_cat::src/test.cpp::1234::whoops",
+      messages.at(1));
+  messages.clear();
+
+  handler.setLevel(LogLevel::DBG2);
+  logMsg(LogLevel::DBG3, "dbg");
+  logMsg(LogLevel::DBG1, "here");
+  logMsg(LogLevel::DBG2, "and here");
+  logMsg(LogLevel::ERR, "oh noes");
+  logMsg(LogLevel::DBG9, "very verbose");
+
+  ASSERT_EQ(3, messages.size());
+  EXPECT_EQ(
+      "LogLevel::DBG1::log_cat::handler_cat::src/test.cpp::1234::here",
+      messages.at(0));
+  EXPECT_EQ(
+      "LogLevel::DBG2::log_cat::handler_cat::src/test.cpp::1234::and here",
+      messages.at(1));
+  EXPECT_EQ(
+      "LogLevel::ERR::log_cat::handler_cat::src/test.cpp::1234::oh noes",
+      messages.at(2));
+  messages.clear();
+}

folly/experimental/logging/test/TestLogHandler.h

@@ -32,7 +32,6 @@ class TestLogHandler : public LogHandler {
     return messages_;
   }
 
- protected:
   void handleMessage(
       const LogMessage& message,
       const LogCategory* handlerCategory) override {