Support negative compression levels

Summary:
Folly uses level [-1, -3] for the default/fastest/best compression levels.
Introduce a `ZSTD_FAST` codec to use negative compression levels.

Reviewed By: chipturner

Differential Revision: D8529340

fbshipit-source-id: 7fb7ff65f08d4dcab009c2eb073920b53055ed9b

folly/compression/Compression.cpp

@@ -1507,6 +1507,22 @@ static int zstdConvertLevel(int level) {
   return level;
 }
 
+static int zstdFastConvertLevel(int level) {
+  switch (level) {
+    case COMPRESSION_LEVEL_FASTEST:
+      return -5;
+    case COMPRESSION_LEVEL_DEFAULT:
+      return -1;
+    case COMPRESSION_LEVEL_BEST:
+      return -1;
+  }
+  if (level < 1) {
+    throw std::invalid_argument(
+        to<std::string>("ZSTD: invalid level: ", level));
+  }
+  return -level;
+}
+
 std::unique_ptr<Codec> getZstdCodec(int level, CodecType type) {
   DCHECK(type == CodecType::ZSTD);
   return zstd::getCodec(zstd::Options(zstdConvertLevel(level)));
@@ -1517,6 +1533,16 @@ std::unique_ptr<StreamCodec> getZstdStreamCodec(int level, CodecType type) {
   return zstd::getStreamCodec(zstd::Options(zstdConvertLevel(level)));
 }
 
+std::unique_ptr<Codec> getZstdFastCodec(int level, CodecType type) {
+  DCHECK(type == CodecType::ZSTD_FAST);
+  return zstd::getCodec(zstd::Options(zstdFastConvertLevel(level)));
+}
+
+std::unique_ptr<StreamCodec> getZstdFastStreamCodec(int level, CodecType type) {
+  DCHECK(type == CodecType::ZSTD_FAST);
+  return zstd::getStreamCodec(zstd::Options(zstdFastConvertLevel(level)));
+}
+
 #endif // FOLLY_HAVE_LIBZSTD
 
 #if FOLLY_HAVE_LIBBZ2
@@ -2027,6 +2053,12 @@ constexpr Factory
 #else
         {},
 #endif
+
+#if FOLLY_HAVE_LIBZSTD
+        {getZstdFastCodec, getZstdFastStreamCodec},
+#else
+        {},
+#endif
 };
 
 Factory const& getFactory(CodecType type) {

folly/compression/Compression.h

@@ -80,6 +80,8 @@ enum class CodecType {
 
   /**
    * Use ZSTD compression.
+   * Levels supported: 1 = fast, ..., 19 = best; default = 3
+   * Use ZSTD_FAST for the fastest zstd compression (negative levels).
    */
   ZSTD = 8,
 
@@ -102,7 +104,20 @@ enum class CodecType {
    */
   BZIP2 = 11,
 
-  NUM_CODEC_TYPES = 12,
+  /**
+   * Use ZSTD compression with a negative compression level (1=-1, 2=-2, ...).
+   * Higher compression levels mean faster.
+   * Level 1 is around the same speed as Snappy with better compression.
+   * Level 5 is around the same speed as LZ4 with slightly worse compression.
+   * Each level gains about 6-15% speed and loses 3-7% compression.
+   * Decompression speed improves for each level, and level 1 decompression
+   * speed is around 25% faster than ZSTD.
+   * This codec is fully compatible with ZSTD.
+   * Levels supported: 1 = best, ..., 5 = fast; default = 1
+   */
+  ZSTD_FAST = 12,
+
+  NUM_CODEC_TYPES = 13,
 };
 
 class Codec {

folly/compression/Zstd.cpp

@@ -120,8 +120,15 @@ bool ZSTDStreamCodec::canUncompress(const IOBuf* data, Optional<uint64_t>)
   return dataStartsWithLE(data, kZSTDMagicLE);
 }
 
+CodecType codecType(Options const& options) {
+  int const level = options.level();
+  DCHECK_NE(level, 0);
+  return level > 0 ? CodecType::ZSTD : CodecType::ZSTD_FAST;
+}
+
 ZSTDStreamCodec::ZSTDStreamCodec(Options options)
-    : StreamCodec(CodecType::ZSTD), options_(std::move(options)) {}
+    : StreamCodec(codecType(options), options.level()),
+      options_(std::move(options)) {}
 
 bool ZSTDStreamCodec::doNeedsUncompressedLength() const {
   return false;
@@ -228,7 +235,7 @@ bool ZSTDStreamCodec::doUncompressStream(
 
 } // namespace
 
-Options::Options(int level) : params_(ZSTD_createCCtxParams()) {
+Options::Options(int level) : params_(ZSTD_createCCtxParams()), level_(level) {
   if (params_ == nullptr) {
     throw std::bad_alloc{};
   }
@@ -238,10 +245,17 @@ Options::Options(int level) : params_(ZSTD_createCCtxParams()) {
   zstdThrowIfError(ZSTD_initCCtxParams(params_.get(), level));
   set(ZSTD_p_contentSizeFlag, 1);
 #endif
+  // zstd-1.3.4 is buggy and only disables Huffman decompression for negative
+  // compression levels if this call is present. This call is begign in other
+  // versions.
+  set(ZSTD_p_compressionLevel, level);
 }
 
 void Options::set(ZSTD_cParameter param, unsigned value) {
   zstdThrowIfError(ZSTD_CCtxParam_setParameter(params_.get(), param, value));
+  if (param == ZSTD_p_compressionLevel) {
+    level_ = static_cast<int>(value);
+  }
 }
 
 /* static */ void Options::freeCCtxParams(ZSTD_CCtx_params* params) {

folly/compression/Zstd.h

@@ -64,6 +64,11 @@ class Options {
     return params_.get();
   }
 
+  /// Get the compression level.
+  int level() const {
+    return level_;
+  }
+
   /// Get the maximum window size.
   size_t maxWindowSize() const {
     return maxWindowSize_;
@@ -76,6 +81,7 @@ class Options {
       folly::static_function_deleter<ZSTD_CCtx_params, &freeCCtxParams>>
       params_;
   size_t maxWindowSize_{0};
+  int level_;
 };
 
 /// Get a zstd Codec with the given options.

folly/compression/test/CompressionTest.cpp

@@ -1408,6 +1408,22 @@ TEST(ZstdTest, CustomOptions) {
   }
 }
 
+TEST(ZstdTest, NegativeLevels) {
+  EXPECT_EQ(zstd::Options(1).level(), 1);
+  EXPECT_EQ(zstd::Options(-1).level(), -1);
+  auto const original = std::string(
+      reinterpret_cast<const char*>(randomDataHolder.data(16348).data()),
+      16348);
+  auto const plusCompressed =
+      zstd::getCodec(zstd::Options(1))->compress(original);
+  auto const minusCompressed =
+      zstd::getCodec(zstd::Options(-100))->compress(original);
+  EXPECT_GT(minusCompressed.size(), plusCompressed.size());
+  auto codec = getCodec(CodecType::ZSTD);
+  auto const uncompressed = codec->uncompress(minusCompressed);
+  EXPECT_EQ(original, uncompressed);
+}
+
 #endif
 
 #if FOLLY_HAVE_LIBZ