Implemented the Blake2xb XOF (extendable output function)

Summary: Implemented the Blake2xb XOF. See the specification at https://blake2.net/blake2x.pdf.

Reviewed By: djwatson

Differential Revision: D13403920

fbshipit-source-id: 38afd6d3b782c2f6649365d01ca3d83b55c90c4b

CMake/FindLibsodium.cmake

+find_path(LIBSODIUM_INCLUDE_DIR NAMES sodium.h)
+mark_as_advanced(LIBSODIUM_INCLUDE_DIR)
+
+find_library(LIBSODIUM_LIBRARY NAMES sodium)
+mark_as_advanced(LIBSODIUM_LIBRARY)
+
+include(FindPackageHandleStandardArgs)
+FIND_PACKAGE_HANDLE_STANDARD_ARGS(
+  LIBSODIUM
+  REQUIRED_VARS LIBSODIUM_LIBRARY LIBSODIUM_INCLUDE_DIR)
+
+if(LIBSODIUM_FOUND)
+  set(LIBSODIUM_LIBRARIES ${LIBSODIUM_LIBRARY})
+  set(LIBSODIUM_INCLUDE_DIRS ${LIBSODIUM_INCLUDE_DIR})
+  message(STATUS "Found Libsodium: ${LIBSODIUM_LIBRARY}")
+endif()

CMake/folly-deps.cmake

@@ -128,6 +128,10 @@ find_package(LibAIO)
 list(APPEND FOLLY_LINK_LIBRARIES ${LIBAIO_LIBRARIES})
 list(APPEND FOLLY_INCLUDE_DIRECTORIES ${LIBAIO_INCLUDE_DIRS})
 
+find_package(Libsodium)
+list(APPEND FOLLY_LINK_LIBRARIES ${LIBSODIUM_LIBRARIES})
+list(APPEND FOLLY_INCLUDE_DIRECTORIES ${LIBSODIUM_INCLUDE_DIRS})
+
 list(APPEND FOLLY_LINK_LIBRARIES ${CMAKE_DL_LIBS})
 list(APPEND CMAKE_REQUIRED_LIBRARIES ${CMAKE_DL_LIBS})
 

CMakeLists.txt

@@ -170,6 +170,14 @@ if (NOT ${LIBAIO_FOUND})
     ${FOLLY_DIR}/experimental/io/AsyncIO.h
   )
 endif()
+if (NOT ${LIBSODIUM_FOUND})
+  list(REMOVE_ITEM files
+    ${FOLLY_DIR}/experimental/crypto/Blake2xb.cpp
+  )
+  list(REMOVE_ITEM hfiles
+    ${FOLLY_DIR}/experimental/crypto/Blake2xb.h
+  )
+endif()
 if(CMAKE_SYSTEM_NAME STREQUAL "Windows")
   list(REMOVE_ITEM files
     ${FOLLY_DIR}/Poly.cpp

folly/experimental/crypto/Blake2xb.cpp

+/*
+ * Copyright 2017-present Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <array>
+
+#include <folly/experimental/crypto/Blake2xb.h>
+#include <folly/lang/Bits.h>
+
+namespace folly {
+namespace crypto {
+
+// static
+constexpr size_t Blake2xb::kMinOutputLength;
+// static
+constexpr size_t Blake2xb::kMaxOutputLength;
+// static
+constexpr size_t Blake2xb::kUnknownOutputLength;
+
+namespace {
+constexpr std::array<uint64_t, 8> kBlake2bIV = {{
+    0x6a09e667f3bcc908ULL,
+    0xbb67ae8584caa73bULL,
+    0x3c6ef372fe94f82bULL,
+    0xa54ff53a5f1d36f1ULL,
+    0x510e527fade682d1ULL,
+    0x9b05688c2b3e6c1fULL,
+    0x1f83d9abfb41bd6bULL,
+    0x5be0cd19137e2179ULL,
+}};
+
+void initStateFromParams(
+    crypto_generichash_blake2b_state* state,
+    const detail::Blake2xbParam& param,
+    ByteRange key) {
+  const uint64_t* p = reinterpret_cast<const uint64_t*>(&param);
+  for (int i = 0; i < 8; ++i) {
+    state->h[i] = kBlake2bIV.data()[i] ^ Endian::little(p[i]);
+  }
+  std::memset(
+      reinterpret_cast<uint8_t*>(state) + sizeof(state->h),
+      0,
+      sizeof(*state) - sizeof(state->h));
+  if (key.size() > 0) {
+    if (key.size() < crypto_generichash_blake2b_KEYBYTES_MIN ||
+        key.size() > crypto_generichash_blake2b_KEYBYTES_MAX) {
+      throw std::runtime_error("invalid key size");
+    }
+    std::array<uint8_t, 128> block;
+    memcpy(block.data(), key.data(), key.size());
+    memset(block.data() + key.size(), 0, block.size() - key.size());
+    crypto_generichash_blake2b_update(state, block.data(), block.size());
+    sodium_memzero(block.data(), block.size()); // erase key from stack
+  }
+}
+} // namespace
+
+Blake2xb::Blake2xb()
+    : param_{},
+      state_{},
+      outputLengthKnown_{false},
+      initialized_{false},
+      finished_{false} {
+  static const int sodiumInitResult = sodium_init();
+  if (sodiumInitResult == -1) {
+    throw std::runtime_error("sodium_init() failed");
+  }
+}
+
+Blake2xb::~Blake2xb() = default;
+
+void Blake2xb::init(
+    size_t outputLength,
+    ByteRange key /* = {} */,
+    ByteRange salt /* = {} */,
+    ByteRange personalization /* = {}*/) {
+  if (outputLength == kUnknownOutputLength) {
+    outputLengthKnown_ = false;
+    outputLength = kUnknownOutputLengthMagic;
+  } else if (outputLength > kMaxOutputLength) {
+    throw std::runtime_error("Output length too large");
+  } else {
+    outputLengthKnown_ = true;
+  }
+  std::memset(&param_, 0, sizeof(param_));
+  param_.digestLength = crypto_generichash_blake2b_BYTES_MAX;
+  param_.keyLength = static_cast<uint8_t>(key.size());
+  param_.fanout = 1;
+  param_.depth = 1;
+  param_.xofLength = Endian::little(static_cast<uint32_t>(outputLength));
+  if (salt.size() > 0) {
+    if (salt.size() != crypto_generichash_blake2b_SALTBYTES) {
+      throw std::runtime_error("Invalid salt length, must be 16 bytes");
+    }
+    std::memcpy(param_.salt, salt.data(), sizeof(param_.salt));
+  }
+  if (personalization.size() > 0) {
+    if (personalization.size() != crypto_generichash_blake2b_PERSONALBYTES) {
+      throw std::runtime_error(
+          "Invalid personalization length, must be 16 bytes");
+    }
+    std::memcpy(
+        param_.personal, personalization.data(), sizeof(param_.personal));
+  }
+  initStateFromParams(&state_, param_, key);
+  initialized_ = true;
+  finished_ = false;
+}
+
+void Blake2xb::update(ByteRange data) {
+  if (!initialized_) {
+    throw std::runtime_error("Must call init() before calling update()");
+  } else if (finished_) {
+    throw std::runtime_error("Can't call update() after finish()");
+  }
+  int res =
+      crypto_generichash_blake2b_update(&state_, data.data(), data.size());
+  if (res != 0) {
+    throw std::runtime_error("crypto_generichash_blake2b_update() failed");
+  }
+}
+
+void Blake2xb::finish(MutableByteRange out) {
+  if (!initialized_) {
+    throw std::runtime_error("Must call init() before calling finish()");
+  } else if (finished_) {
+    throw std::runtime_error("finish() already called");
+  }
+
+  if (outputLengthKnown_) {
+    uint32_t outLength = static_cast<uint32_t>(out.size());
+    if (outLength != Endian::little(param_.xofLength)) {
+      throw std::runtime_error("out.size() must equal output length");
+    }
+  }
+
+  std::array<uint8_t, crypto_generichash_blake2b_BYTES_MAX> h0;
+  int res = crypto_generichash_blake2b_final(&state_, h0.data(), h0.size());
+  if (res != 0) {
+    throw std::runtime_error("crypto_generichash_blake2b_final() failed");
+  }
+
+  param_.keyLength = 0;
+  param_.fanout = 0;
+  param_.depth = 0;
+  param_.leafLength = Endian::little(
+      static_cast<uint32_t>(crypto_generichash_blake2b_BYTES_MAX));
+  param_.innerLength = crypto_generichash_blake2b_BYTES_MAX;
+  size_t pos = 0;
+  size_t remaining = out.size();
+  while (remaining > 0) {
+    param_.nodeOffset = Endian::little(
+        static_cast<uint32_t>(pos / crypto_generichash_blake2b_BYTES_MAX));
+    size_t len = std::min(
+        static_cast<size_t>(crypto_generichash_blake2b_BYTES_MAX), remaining);
+    param_.digestLength = static_cast<uint8_t>(len);
+    initStateFromParams(&state_, param_, {} /* key */);
+    res = crypto_generichash_blake2b_update(&state_, h0.data(), h0.size());
+    if (res != 0) {
+      throw std::runtime_error("crypto_generichash_blake2b_update() failed");
+    }
+    res = crypto_generichash_blake2b_final(&state_, out.data() + pos, len);
+    if (res != 0) {
+      throw std::runtime_error("crypto_generichash_blake2b_final() failed");
+    }
+    pos += len;
+    remaining -= len;
+  }
+  finished_ = true;
+}
+
+} // namespace crypto
+} // namespace folly

folly/experimental/crypto/Blake2xb.h

+/*
+ * Copyright 2017-present Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <sodium.h>
+
+#include <folly/Range.h>
+
+namespace folly {
+namespace crypto {
+
+namespace detail {
+
+struct Blake2xbParam {
+  uint8_t digestLength; /*  1 */
+  uint8_t keyLength; /*  2 */
+  uint8_t fanout; /*  3 */
+  uint8_t depth; /*  4 */
+  uint32_t leafLength; /*  8 */
+  uint32_t nodeOffset; /* 12 */
+  uint32_t xofLength; /* 16 */
+  uint8_t nodeDepth; /* 17 */
+  uint8_t innerLength; /* 18 */
+  uint8_t reserved[14]; /* 32 */
+  uint8_t salt[16]; /* 48 */
+  uint8_t personal[16]; /* 64 */
+};
+
+static_assert(sizeof(Blake2xbParam) == 64, "wrong sizeof(Blake2xbParam)");
+
+} // namespace detail
+
+/**
+ * An implementation of the BLAKE2x XOF (extendable output function)
+ * hash function using BLAKE2b as the underlying hash. This hash function
+ * can produce cryptographic hashes of arbitrary length (between 1 and 2^32 - 2
+ * bytes) from inputs of arbitrary size. Like BLAKE2b, it can be keyed, and can
+ * accept optional salt and personlization parameters.
+ *
+ * Note that if you need to compute hashes between 16 and 64 bytes in length,
+ * you should use Blake2b instead - it's more efficient and you will have an
+ * easier time interoperating with other languages, since implementations of
+ * Blake2b are more common than implementations of Blake2xb. You can generate
+ * a blake2b hash using the following functions from libsodium:
+ * - crypto_generichash_blake2b()
+ * - crypto_generichash_blake2b_salt_personal()
+ */
+class Blake2xb {
+ public:
+  /**
+   * Minimum output hash size, if it is known in advance.
+   */
+  static constexpr size_t kMinOutputLength = 1;
+  /**
+   * Maximum output hash size, if it is known in advance.
+   */
+  static constexpr size_t kMaxOutputLength = 0xfffffffeULL;
+  /**
+   * If the amount of output data desired is not known in advance, use this
+   * constant as the outputLength parameter to init() or reinit().
+   */
+  static constexpr size_t kUnknownOutputLength = 0;
+
+  /**
+   * Creates a new uninitialized Blake2xb instance. The init() method must
+   * be called before it can be used.
+   */
+  Blake2xb();
+
+  /**
+   * Shorthand for calling the no-argument constructor followed by
+   * newInstance.init(outputLength, key, salt, personlization).
+   */
+  explicit Blake2xb(
+      size_t outputLength,
+      ByteRange key = {},
+      ByteRange salt = {},
+      ByteRange personalization = {})
+      : Blake2xb() {
+    init(outputLength, key, salt, personalization);
+  }
+
+  ~Blake2xb();
+
+  /**
+   * Initializes the digest object. This must be called after a new instance
+   * is constructed and before update() is called. It can also be called on
+   * a previously-used instance to reset its internal state and reuse it for
+   * a new hash computation.
+   */
+  void init(
+      size_t outputLength,
+      ByteRange key = {},
+      ByteRange salt = {},
+      ByteRange personalization = {});
+
+  /**
+   * Hashes some more input data.
+   */
+  void update(ByteRange data);
+
+  /**
+   * Computes the final hash and stores it in the given output. The value of
+   * out.size() MUST equal the outputLength parameter that was given to the
+   * last init() or reinit() call, except when the outputLength parameter was
+   * kUnknownOutputLength.
+   *
+   * WARNING: never compare the results of two Blake2xb.finish() calls
+   * using non-constant time comparison. The recommended way to compare
+   * cryptographic hashes is with sodium_memcmp() (or some other constant-time
+   * memory comparison function).
+   */
+  void finish(MutableByteRange out);
+
+  /**
+   * Convenience function, use this if you are hashing a single input buffer,
+   * the output length is known in advance, and the output data is allocated
+   * and ready to accept the hash value.
+   */
+  static void hash(
+      MutableByteRange out,
+      ByteRange data,
+      ByteRange key = {},
+      ByteRange salt = {},
+      ByteRange personalization = {}) {
+    Blake2xb d;
+    d.init(out.size(), key, salt, personalization);
+    d.update(data);
+    d.finish(out);
+  }
+
+ private:
+  static constexpr size_t kUnknownOutputLengthMagic = 0xffffffffULL;
+
+  detail::Blake2xbParam param_;
+  crypto_generichash_blake2b_state state_;
+  bool outputLengthKnown_;
+  bool initialized_;
+  bool finished_;
+};
+
+} // namespace crypto
+} // namespace folly

folly/experimental/crypto/test/Blake2xbBenchmark.cpp

+/*
+ * Copyright 2017-present Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <folly/experimental/crypto/Blake2xb.h>
+
+#include <vector>
+
+#include <sodium.h>
+
+#include <folly/Benchmark.h>
+#include <folly/Random.h>
+#include <folly/init/Init.h>
+#include <folly/io/IOBuf.h>
+
+#include <glog/logging.h>
+
+using namespace ::folly::crypto;
+
+void benchmarkBlake2b(size_t inputSize, size_t n) {
+  std::array<uint8_t, crypto_generichash_blake2b_BYTES_MAX> result;
+  std::vector<uint8_t> input;
+  BENCHMARK_SUSPEND {
+    input.resize(inputSize);
+  };
+  for (size_t i = 0; i < static_cast<size_t>(n); ++i) {
+    int res = crypto_generichash_blake2b(
+        result.data(), sizeof(result), input.data(), input.size(), nullptr, 0);
+    if (res != 0) {
+      throw std::runtime_error("blake2b hash failed");
+    }
+  }
+}
+
+void benchmarkBlake2bMultiple(size_t inputSize, size_t m, size_t n) {
+  std::vector<uint8_t> input;
+  std::vector<uint8_t> output;
+  std::vector<uint8_t> personalization;
+  std::array<uint8_t, crypto_generichash_blake2b_BYTES_MAX> h0;
+  BENCHMARK_SUSPEND {
+    output.resize(crypto_generichash_blake2b_BYTES_MAX * m);
+    input.resize(inputSize);
+    personalization.resize(crypto_generichash_blake2b_PERSONALBYTES);
+  };
+  for (size_t i = 0; i < static_cast<size_t>(n); ++i) {
+    int res = crypto_generichash_blake2b(
+        h0.data(), sizeof(h0), input.data(), input.size(), nullptr, 0);
+    if (res != 0) {
+      throw std::runtime_error("blake2b hash failed");
+    }
+
+    for (size_t j = 0; j < m; j++) {
+      res = crypto_generichash_blake2b_salt_personal(
+          output.data() + (crypto_generichash_blake2b_BYTES_MAX * j),
+          crypto_generichash_blake2b_BYTES_MAX,
+          h0.data(),
+          h0.size(),
+          nullptr /* key */,
+          0 /* keylen */,
+          nullptr /* salt */,
+          personalization.data());
+      if (res != 0) {
+        throw std::runtime_error("blake2b hash failed");
+      }
+      sodium_increment(
+          personalization.data(), crypto_generichash_blake2b_PERSONALBYTES);
+    }
+  }
+}
+
+void benchmarkBlake2xb(size_t inputSize, size_t outputSize, size_t n) {
+  std::vector<uint8_t> input;
+  std::vector<uint8_t> output;
+  BENCHMARK_SUSPEND {
+    input.resize(inputSize);
+    output.resize(outputSize);
+  };
+  for (size_t i = 0; i < static_cast<size_t>(n); ++i) {
+    Blake2xb::hash({output.data(), output.size()}, folly::range(input));
+  }
+}
+
+BENCHMARK(blake2b_100b_in_64b_out, n) {
+  benchmarkBlake2b(100, n);
+}
+
+BENCHMARK_RELATIVE(blake2xb_100b_in_64b_out, n) {
+  benchmarkBlake2xb(100, 64, n);
+}
+
+BENCHMARK(blake2b_100b_in_128b_out, n) {
+  benchmarkBlake2bMultiple(100, 128 / 64, n);
+}
+
+BENCHMARK_RELATIVE(blake2xb_100b_in_128b_out, n) {
+  benchmarkBlake2xb(100, 128, n);
+}
+
+BENCHMARK(blake2b_100b_in_1024b_out, n) {
+  benchmarkBlake2bMultiple(100, 1024 / 64, n);
+}
+
+BENCHMARK_RELATIVE(blake2xb_100b_in_1024b_out, n) {
+  benchmarkBlake2xb(100, 1024, n);
+}
+
+BENCHMARK(blake2b_100b_in_4096b_out, n) {
+  benchmarkBlake2bMultiple(100, 4096 / 64, n);
+}
+
+BENCHMARK_RELATIVE(blake2xb_100b_in_4096b_out, n) {
+  benchmarkBlake2xb(100, 4096, n);
+}
+
+BENCHMARK(blake2b_1000b_in_64b_out, n) {
+  benchmarkBlake2b(1000, n);
+}
+
+BENCHMARK_RELATIVE(blake2xb_1000b_in_64b_out, n) {
+  benchmarkBlake2xb(1000, 64, n);
+}
+
+BENCHMARK(blake2b_1000b_in_128b_out, n) {
+  benchmarkBlake2bMultiple(1000, 128 / 64, n);
+}
+
+BENCHMARK_RELATIVE(blake2xb_1000b_in_128b_out, n) {
+  benchmarkBlake2xb(1000, 128, n);
+}
+
+BENCHMARK(blake2b_1000b_in_1024b_out, n) {
+  benchmarkBlake2bMultiple(1000, 1024 / 64, n);
+}
+
+BENCHMARK_RELATIVE(blake2xb_1000b_in_1024b_out, n) {
+  benchmarkBlake2xb(1000, 1024, n);
+}
+
+BENCHMARK(blake2b_1000b_in_4096b_out, n) {
+  benchmarkBlake2bMultiple(1000, 4096 / 64, n);
+}
+
+BENCHMARK_RELATIVE(blake2xb_1000b_in_4096b_out, n) {
+  benchmarkBlake2xb(1000, 4096, n);
+}
+
+int main(int argc, char** argv) {
+  folly::init(&argc, &argv);
+  folly::runBenchmarks();
+  return 0;
+}