Add zlib-specific codec initialization

Summary:
- Create interface to initialize zlib codec using specific parameters
- This enables the raw inflate/deflate and auto inflate options
- Add tests for option initialization

Reviewed By: terrelln, yfeldblum

Differential Revision: D5649980

fbshipit-source-id: fd36e8edc0e8c528cd6c9d8f39e8ef839b6acfef

folly/Makefile.am

@@ -298,6 +298,8 @@ nobase_follyinclude_HEADERS = \
 	io/async/test/TimeUtil.h \
 	io/async/test/UndelayedDestruction.h \
 	io/async/test/Util.h \
+	io/compression/Utils.h \
+	io/compression/Zlib.h \
 	Iterator.h \
 	json.h \
 	Launder.h \
@@ -533,6 +535,7 @@ libfolly_la_SOURCES = \
 	io/async/test/TimeUtil.cpp \
 	io/async/ssl/OpenSSLUtils.cpp \
 	io/async/ssl/SSLErrors.cpp \
+	io/compression/Zlib.cpp \
 	json.cpp \
 	detail/MemoryIdler.cpp \
 	detail/SocketFastOpen.cpp \

folly/io/compression/Utils.h

+/*
+ * Copyright 2017 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <type_traits>
+
+#include <folly/Bits.h>
+#include <folly/io/Cursor.h>
+#include <folly/io/IOBuf.h>
+
+/**
+ * Helper functions for compression codecs.
+ */
+namespace folly {
+namespace io {
+namespace compression {
+namespace detail {
+
+/**
+ * Reads sizeof(T) bytes, and returns false if not enough bytes are available.
+ * Returns true if the first n bytes are equal to prefix when interpreted as
+ * a little endian T.
+ */
+template <typename T>
+typename std::enable_if<std::is_unsigned<T>::value, bool>::type
+dataStartsWithLE(const IOBuf* data, T prefix, uint64_t n = sizeof(T)) {
+  DCHECK_GT(n, 0);
+  DCHECK_LE(n, sizeof(T));
+  T value;
+  Cursor cursor{data};
+  if (!cursor.tryReadLE(value)) {
+    return false;
+  }
+  const T mask = n == sizeof(T) ? T(-1) : (T(1) << (8 * n)) - 1;
+  return prefix == (value & mask);
+}
+
+template <typename T>
+typename std::enable_if<std::is_arithmetic<T>::value, std::string>::type
+prefixToStringLE(T prefix, uint64_t n = sizeof(T)) {
+  DCHECK_GT(n, 0);
+  DCHECK_LE(n, sizeof(T));
+  prefix = Endian::little(prefix);
+  std::string result;
+  result.resize(n);
+  memcpy(&result[0], &prefix, n);
+  return result;
+}
+
+} // namespace detail
+} // namespace compression
+} // namespace io
+} // namespace folly

folly/io/compression/Zlib.h

+/*
+ * Copyright 2017 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <folly/Portability.h>
+#include <folly/io/Compression.h>
+
+#if FOLLY_HAVE_LIBZ
+
+#include <zlib.h>
+
+/**
+ * Interface for Zlib-specific codec initialization.
+ */
+namespace folly {
+namespace io {
+namespace zlib {
+
+struct Options {
+  /**
+   * ZLIB: default option -- write a zlib wrapper as documented in RFC 1950.
+   *
+   * GZIP: write a simple gzip header and trailer around the compressed data
+   * instead of a zlib wrapper.
+   *
+   * RAW: deflate will generate raw deflate data with no zlib header or
+   * trailer, and will not compute a check value.
+   *
+   * AUTO: enable automatic header detection for decoding gzip or zlib data.
+   * For deflation, ZLIB will be used.
+   */
+  enum class Format { ZLIB, GZIP, RAW, AUTO };
+
+  explicit Options(
+      Format format = Format::ZLIB,
+      int windowSize = 15,
+      int memLevel = 8,
+      int strategy = Z_DEFAULT_STRATEGY)
+      : format(format),
+        windowSize(windowSize),
+        memLevel(memLevel),
+        strategy(strategy) {}
+
+  Format format;
+
+  /**
+   * windowSize is the base two logarithm of the window size (the size of the
+   * history buffer). It should be in the range 9..15. Larger values of this
+   * parameter result in better compression at the expense of memory usage.
+   *
+   * The default value is 15.
+   *
+   * NB: when inflating/uncompressing data, the windowSize must be greater than
+   * or equal to the size used when deflating/compressing.
+   */
+  int windowSize;
+
+  /**
+   * "The memLevel parameter specifies how much memory should be allocated for
+   * the internal compression state. memLevel=1 uses minimum memory but is slow
+   * and reduces compression ratio; memLevel=9 uses maximum memory for optimal
+   * speed. The default value is 8."
+   */
+  int memLevel;
+
+  /**
+   * The strategy parameter is used to tune the compression algorithm.
+   * Supported values:
+   * - Z_DEFAULT_STRATEGY: normal data
+   * - Z_FILTERED: data produced by a filter (or predictor)
+   * - Z_HUFFMAN_ONLY: force Huffman encoding only (no string match)
+   * - Z_RLE: limit match distances to one
+   * - Z_FIXED: prevents the use of dynamic Huffman codes
+   *
+   * The strategy parameter only affects the compression ratio but not the
+   * correctness of the compressed output.
+   */
+  int strategy;
+};
+
+/**
+ * Get the default options for gzip compression.
+ * A codec created with these options will have type CodecType::GZIP.
+ */
+Options defaultGzipOptions();
+
+/**
+ * Get the default options for zlib compression.
+ * A codec created with these options will have type CodecType::ZLIB.
+ */
+Options defaultZlibOptions();
+
+/**
+ * Get a codec with the given options and compression level.
+ *
+ * If the windowSize is 15 and the format is Format::ZLIB or Format::GZIP, then
+ * the type of the codec will be CodecType::ZLIB or CodecType::GZIP
+ * respectively. Otherwise, the type will be CodecType::USER_DEFINED.
+ *
+ * Automatic uncompression is not supported with USER_DEFINED codecs.
+ *
+ * Levels supported: 0 = no compression, 1 = fast, ..., 9 = best; default = 6
+ */
+std::unique_ptr<Codec> getCodec(
+    Options options = Options(),
+    int level = COMPRESSION_LEVEL_DEFAULT);
+std::unique_ptr<StreamCodec> getStreamCodec(
+    Options options = Options(),
+    int level = COMPRESSION_LEVEL_DEFAULT);
+
+} // namespace zlib
+} // namespace io
+} // namespace folly
+
+#endif // FOLLY_HAVE_LIBZ

folly/io/test/CompressionTest.cpp

@@ -38,6 +38,12 @@
 #include <zstd.h>
 #endif
 
+#if FOLLY_HAVE_LIBZ
+#include <folly/io/compression/Zlib.h>
+#endif
+
+namespace zlib = folly::io::zlib;
+
 namespace folly {
 namespace io {
 namespace test {
@@ -1129,6 +1135,118 @@ TEST(ZstdTest, BackwardCompatible) {
 }
 
 #endif
+
+#if FOLLY_HAVE_LIBZ
+
+using ZlibFormat = zlib::Options::Format;
+
+TEST(ZlibTest, Auto) {
+  size_t const uncompressedLength_ = (size_t)1 << 15;
+  auto const original = std::string(
+      reinterpret_cast<const char*>(
+          randomDataHolder.data(uncompressedLength_).data()),
+      uncompressedLength_);
+  auto optionCodec = zlib::getCodec(zlib::Options(ZlibFormat::AUTO));
+
+  // Test the codec can uncompress zlib data.
+  {
+    auto codec = getCodec(CodecType::ZLIB);
+    auto const compressed = codec->compress(original);
+    auto const uncompressed = optionCodec->uncompress(compressed);
+    EXPECT_EQ(original, uncompressed);
+  }
+
+  // Test the codec can uncompress gzip data.
+  {
+    auto codec = getCodec(CodecType::GZIP);
+    auto const compressed = codec->compress(original);
+    auto const uncompressed = optionCodec->uncompress(compressed);
+    EXPECT_EQ(original, uncompressed);
+  }
+}
+
+TEST(ZlibTest, DefaultOptions) {
+  size_t const uncompressedLength_ = (size_t)1 << 20;
+  auto const original = std::string(
+      reinterpret_cast<const char*>(
+          randomDataHolder.data(uncompressedLength_).data()),
+      uncompressedLength_);
+  {
+    auto codec = getCodec(CodecType::ZLIB);
+    auto optionCodec = zlib::getCodec(zlib::defaultZlibOptions());
+    auto const compressed = optionCodec->compress(original);
+    auto uncompressed = codec->uncompress(compressed);
+    EXPECT_EQ(original, uncompressed);
+    uncompressed = optionCodec->uncompress(compressed);
+    EXPECT_EQ(original, uncompressed);
+  }
+
+  {
+    auto codec = getCodec(CodecType::GZIP);
+    auto optionCodec = zlib::getCodec(zlib::defaultGzipOptions());
+    auto const compressed = optionCodec->compress(original);
+    auto uncompressed = codec->uncompress(compressed);
+    EXPECT_EQ(original, uncompressed);
+    uncompressed = optionCodec->uncompress(compressed);
+    EXPECT_EQ(original, uncompressed);
+  }
+}
+
+class ZlibOptionsTest : public testing::TestWithParam<
+                            std::tr1::tuple<ZlibFormat, int, int, int>> {
+ protected:
+  void SetUp() override {
+    auto tup = GetParam();
+    options_.format = std::tr1::get<0>(tup);
+    options_.windowSize = std::tr1::get<1>(tup);
+    options_.memLevel = std::tr1::get<2>(tup);
+    options_.strategy = std::tr1::get<3>(tup);
+    codec_ = zlib::getStreamCodec(options_);
+  }
+
+  void runSimpleRoundTripTest(const DataHolder& dh);
+
+ private:
+  zlib::Options options_;
+  std::unique_ptr<StreamCodec> codec_;
+};
+
+void ZlibOptionsTest::runSimpleRoundTripTest(const DataHolder& dh) {
+  size_t const uncompressedLength = (size_t)1 << 16;
+  auto const original = std::string(
+      reinterpret_cast<const char*>(dh.data(uncompressedLength).data()),
+      uncompressedLength);
+
+  auto const compressed = codec_->compress(original);
+  auto const uncompressed = codec_->uncompress(compressed);
+  EXPECT_EQ(uncompressed, original);
+}
+
+TEST_P(ZlibOptionsTest, simpleRoundTripTest) {
+  runSimpleRoundTripTest(constantDataHolder);
+  runSimpleRoundTripTest(randomDataHolder);
+}
+
+INSTANTIATE_TEST_CASE_P(
+    ZlibOptionsTest,
+    ZlibOptionsTest,
+    testing::Combine(
+        testing::Values(
+            ZlibFormat::ZLIB,
+            ZlibFormat::GZIP,
+            ZlibFormat::RAW,
+            ZlibFormat::AUTO),
+        testing::Values(9, 12, 15),
+        testing::Values(1, 8, 9),
+        testing::Values(
+            Z_DEFAULT_STRATEGY,
+            Z_FILTERED,
+            Z_HUFFMAN_ONLY,
+            Z_RLE,
+            Z_FIXED)));
+
+#endif // FOLLY_HAVE_LIBZ
+
 } // namespace test
 } // namespace io
 } // namespace folly