Skip to content

F
folly
Commit 9dc004f8 authored by Stella Lau's avatar Stella Lau Committed by Facebook Github Bot
Browse files
Options

Add LZMA streaming interface 

Summary:
- Replace LZMA2Codec with LZMA2StreamCodec
- Update tests to reflect LZMA2_VARINT_SIZE requiring data length

Reviewed By: terrelln

Differential Revision: D5625388

fbshipit-source-id: 3303c6dda5d41f40615c87504a46923815b0b716
parent 4d4a549d
Show whitespace changes
Inline Side-by-side
Showing with 377 additions and 228 deletions
folly/io/Compression.cpp
View file @ 9dc004f8
...@@ -251,6 +251,9 @@ bool StreamCodec::compressStream( ...@@ -251,6 +251,9 @@ bool StreamCodec::compressStream(
} }
return true; return true;
} }
if (!uncompressedLength() && needsDataLength()) {
throw std::runtime_error("Codec: uncompressed length required");
}
if (state_ == State::RESET && !input.empty() && if (state_ == State::RESET && !input.empty() &&
uncompressedLength() == uint64_t(0)) { uncompressedLength() == uint64_t(0)) {
throw std::runtime_error("Codec: invalid uncompressed length"); throw std::runtime_error("Codec: invalid uncompressed length");
...@@ -983,44 +986,71 @@ std::unique_ptr<IOBuf> SnappyCodec::doUncompress( ...@@ -983,44 +986,71 @@ std::unique_ptr<IOBuf> SnappyCodec::doUncompress(
/** /**
* LZMA2 compression * LZMA2 compression
*/ */
class LZMA2Codec final : public Codec { class LZMA2StreamCodec final : public StreamCodec {
public: public:
static std::unique_ptr<Codec> create(int level, CodecType type); static std::unique_ptr<Codec> createCodec(int level, CodecType type);
explicit LZMA2Codec(int level, CodecType type); static std::unique_ptr<StreamCodec> createStream(int level, CodecType type);
explicit LZMA2StreamCodec(int level, CodecType type);
~LZMA2StreamCodec() override;
std::vector<std::string> validPrefixes() const override; std::vector<std::string> validPrefixes() const override;
bool canUncompress(const IOBuf* data, Optional<uint64_t> uncompressedLength) bool canUncompress(const IOBuf* data, Optional<uint64_t> uncompressedLength)
const override; const override;
private: private:
bool doNeedsUncompressedLength() const override; bool doNeedsDataLength() const override;
uint64_t doMaxUncompressedLength() const override; uint64_t doMaxUncompressedLength() const override;
uint64_t doMaxCompressedLength(uint64_t uncompressedLength) const override; uint64_t doMaxCompressedLength(uint64_t uncompressedLength) const override;
bool encodeSize() const { return type() == CodecType::LZMA2_VARINT_SIZE; } bool encodeSize() const {
return type() == CodecType::LZMA2_VARINT_SIZE;
}
std::unique_ptr<IOBuf> doCompress(const IOBuf* data) override; void doResetStream() override;
std::unique_ptr<IOBuf> doUncompress( bool doCompressStream(
const IOBuf* data, ByteRange& input,
Optional<uint64_t> uncompressedLength) override; MutableByteRange& output,
StreamCodec::FlushOp flushOp) override;
bool doUncompressStream(
ByteRange& input,
MutableByteRange& output,
StreamCodec::FlushOp flushOp) override;
void resetCStream();
void resetDStream();
size_t decodeVarint(ByteRange& input);
bool flushVarintBuffer(MutableByteRange& output);
void resetVarintBuffer();
Optional<lzma_stream> cstream_{};
Optional<lzma_stream> dstream_{};
std::unique_ptr<IOBuf> addOutputBuffer(lzma_stream* stream, size_t length); std::array<uint8_t, kMaxVarintLength64> varintBuffer_;
bool doInflate(lzma_stream* stream, IOBuf* head, size_t bufferLength); ByteRange varintToEncode_;
size_t varintBufferPos_{0};
int level_; int level_;
bool needReset_{true};
bool needDecodeSize_{false};
}; };
static constexpr uint64_t kLZMA2MagicLE = 0x005A587A37FD; static constexpr uint64_t kLZMA2MagicLE = 0x005A587A37FD;
static constexpr unsigned kLZMA2MagicBytes = 6; static constexpr unsigned kLZMA2MagicBytes = 6;
std::vector<std::string> LZMA2Codec::validPrefixes() const { std::vector<std::string> LZMA2StreamCodec::validPrefixes() const {
if (type() == CodecType::LZMA2_VARINT_SIZE) { if (type() == CodecType::LZMA2_VARINT_SIZE) {
return {}; return {};
} }
return {prefixToStringLE(kLZMA2MagicLE, kLZMA2MagicBytes)}; return {prefixToStringLE(kLZMA2MagicLE, kLZMA2MagicBytes)};
} }
bool LZMA2Codec::canUncompress(const IOBuf* data, Optional<uint64_t>) const { bool LZMA2StreamCodec::doNeedsDataLength() const {
return encodeSize();
}
bool LZMA2StreamCodec::canUncompress(const IOBuf* data, Optional<uint64_t>)
const {
if (type() == CodecType::LZMA2_VARINT_SIZE) { if (type() == CodecType::LZMA2_VARINT_SIZE) {
return false; return false;
} }
...@@ -1029,11 +1059,20 @@ bool LZMA2Codec::canUncompress(const IOBuf* data, Optional<uint64_t>) const { ...@@ -1029,11 +1059,20 @@ bool LZMA2Codec::canUncompress(const IOBuf* data, Optional<uint64_t>) const {
return dataStartsWithLE(data, kLZMA2MagicLE, kLZMA2MagicBytes); return dataStartsWithLE(data, kLZMA2MagicLE, kLZMA2MagicBytes);
} }
std::unique_ptr<Codec> LZMA2Codec::create(int level, CodecType type) { std::unique_ptr<Codec> LZMA2StreamCodec::createCodec(
return std::make_unique<LZMA2Codec>(level, type); int level,
CodecType type) {
return make_unique<LZMA2StreamCodec>(level, type);
} }
LZMA2Codec::LZMA2Codec(int level, CodecType type) : Codec(type) { std::unique_ptr<StreamCodec> LZMA2StreamCodec::createStream(
int level,
CodecType type) {
return make_unique<LZMA2StreamCodec>(level, type);
}
LZMA2StreamCodec::LZMA2StreamCodec(int level, CodecType type)
: StreamCodec(type) {
DCHECK(type == CodecType::LZMA2 || type == CodecType::LZMA2_VARINT_SIZE); DCHECK(type == CodecType::LZMA2 || type == CodecType::LZMA2_VARINT_SIZE);
switch (level) { switch (level) {
case COMPRESSION_LEVEL_FASTEST: case COMPRESSION_LEVEL_FASTEST:
...@@ -1047,201 +1086,230 @@ LZMA2Codec::LZMA2Codec(int level, CodecType type) : Codec(type) { ...@@ -1047,201 +1086,230 @@ LZMA2Codec::LZMA2Codec(int level, CodecType type) : Codec(type) {
break; break;
} }
if (level < 0 || level > 9) { if (level < 0 || level > 9) {
throw std::invalid_argument(to<std::string>( throw std::invalid_argument(
"LZMA2Codec: invalid level: ", level)); to<std::string>("LZMA2Codec: invalid level: ", level));
} }
level_ = level; level_ = level;
} }
bool LZMA2Codec::doNeedsUncompressedLength() const { LZMA2StreamCodec::~LZMA2StreamCodec() {
return false; if (cstream_) {
lzma_end(cstream_.get_pointer());
cstream_.clear();
}
if (dstream_) {
lzma_end(dstream_.get_pointer());
dstream_.clear();
}
} }
uint64_t LZMA2Codec::doMaxUncompressedLength() const { uint64_t LZMA2StreamCodec::doMaxUncompressedLength() const {
// From lzma/base.h: "Stream is roughly 8 EiB (2^63 bytes)" // From lzma/base.h: "Stream is roughly 8 EiB (2^63 bytes)"
return uint64_t(1) << 63; return uint64_t(1) << 63;
} }
uint64_t LZMA2Codec::doMaxCompressedLength(uint64_t uncompressedLength) const { uint64_t LZMA2StreamCodec::doMaxCompressedLength(
uint64_t uncompressedLength) const {
return lzma_stream_buffer_bound(uncompressedLength) + return lzma_stream_buffer_bound(uncompressedLength) +
(encodeSize() ? kMaxVarintLength64 : 0); (encodeSize() ? kMaxVarintLength64 : 0);
} }
std::unique_ptr<IOBuf> LZMA2Codec::addOutputBuffer( void LZMA2StreamCodec::doResetStream() {
lzma_stream* stream, needReset_ = true;
size_t length) {
CHECK_EQ(stream->avail_out, 0);
auto buf = IOBuf::create(length);
buf->append(buf->capacity());
stream->next_out = buf->writableData();
stream->avail_out = buf->length();
return buf;
} }
std::unique_ptr<IOBuf> LZMA2Codec::doCompress(const IOBuf* data) { void LZMA2StreamCodec::resetCStream() {
lzma_ret rc; if (!cstream_) {
lzma_stream stream = LZMA_STREAM_INIT; cstream_.assign(LZMA_STREAM_INIT);
rc = lzma_easy_encoder(&stream, level_, LZMA_CHECK_NONE);
if (rc != LZMA_OK) {
throw std::runtime_error(folly::to<std::string>(
"LZMA2Codec: lzma_easy_encoder error: ", rc));
}
SCOPE_EXIT { lzma_end(&stream); };
uint64_t uncompressedLength = data->computeChainDataLength();
uint64_t maxCompressedLength = lzma_stream_buffer_bound(uncompressedLength);
// Max 64MiB in one go
constexpr uint32_t maxSingleStepLength = uint32_t(64) << 20; // 64MiB
constexpr uint32_t defaultBufferLength = uint32_t(4) << 20; // 4MiB
auto out = addOutputBuffer(
&stream,
(maxCompressedLength <= maxSingleStepLength ?
maxCompressedLength :
defaultBufferLength));
if (encodeSize()) {
auto size = IOBuf::createCombined(kMaxVarintLength64);
encodeVarintToIOBuf(uncompressedLength, size.get());
size->appendChain(std::move(out));
out = std::move(size);
}
for (auto& range : *data) {
if (range.empty()) {
continue;
}
stream.next_in = const_cast<uint8_t*>(range.data());
stream.avail_in = range.size();
while (stream.avail_in != 0) {
if (stream.avail_out == 0) {
out->prependChain(addOutputBuffer(&stream, defaultBufferLength));
} }
lzma_ret const rc =
rc = lzma_code(&stream, LZMA_RUN); lzma_easy_encoder(cstream_.get_pointer(), level_, LZMA_CHECK_NONE);
if (rc != LZMA_OK) { if (rc != LZMA_OK) {
throw std::runtime_error(folly::to<std::string>( throw std::runtime_error(folly::to<std::string>(
"LZMA2Codec: lzma_code error: ", rc)); "LZMA2StreamCodec: lzma_easy_encoder error: ", rc));
}
}
} }
}
do { void LZMA2StreamCodec::resetDStream() {
if (stream.avail_out == 0) { if (!dstream_) {
out->prependChain(addOutputBuffer(&stream, defaultBufferLength)); dstream_.assign(LZMA_STREAM_INIT);
} }
lzma_ret const rc = lzma_auto_decoder(
rc = lzma_code(&stream, LZMA_FINISH); dstream_.get_pointer(), std::numeric_limits<uint64_t>::max(), 0);
} while (rc == LZMA_OK); if (rc != LZMA_OK) {
if (rc != LZMA_STREAM_END) {
throw std::runtime_error(folly::to<std::string>( throw std::runtime_error(folly::to<std::string>(
"LZMA2Codec: lzma_code ended with error: ", rc)); "LZMA2StreamCodec: lzma_auto_decoder error: ", rc));
} }
out->prev()->trimEnd(stream.avail_out);
return out;
} }
bool LZMA2Codec::doInflate(lzma_stream* stream, static lzma_ret lzmaThrowOnError(lzma_ret const rc) {
IOBuf* head,
size_t bufferLength) {
if (stream->avail_out == 0) {
head->prependChain(addOutputBuffer(stream, bufferLength));
}
lzma_ret rc = lzma_code(stream, LZMA_RUN);
switch (rc) { switch (rc) {
case LZMA_OK: case LZMA_OK:
break;
case LZMA_STREAM_END: case LZMA_STREAM_END:
return true; case LZMA_BUF_ERROR: // not fatal: returned if no progress was made twice
return rc;
default: default:
throw std::runtime_error(to<std::string>( throw std::runtime_error(
"LZMA2Codec: lzma_code error: ", rc)); to<std::string>("LZMA2StreamCodec: error: ", rc));
} }
return false;
} }
std::unique_ptr<IOBuf> LZMA2Codec::doUncompress( static lzma_action lzmaTranslateFlush(StreamCodec::FlushOp flush) {
const IOBuf* data, switch (flush) {
Optional<uint64_t> uncompressedLength) { case StreamCodec::FlushOp::NONE:
lzma_ret rc; return LZMA_RUN;
lzma_stream stream = LZMA_STREAM_INIT; case StreamCodec::FlushOp::FLUSH:
return LZMA_SYNC_FLUSH;
rc = lzma_auto_decoder(&stream, std::numeric_limits<uint64_t>::max(), 0); case StreamCodec::FlushOp::END:
if (rc != LZMA_OK) { return LZMA_FINISH;
throw std::runtime_error(folly::to<std::string>( default:
"LZMA2Codec: lzma_auto_decoder error: ", rc)); throw std::invalid_argument("LZMA2StreamCodec: Invalid flush");
} }
}
SCOPE_EXIT { lzma_end(&stream); }; /**
* Flushes the varint buffer.
// Max 64MiB in one go * Advances output by the number of bytes written.
constexpr uint32_t maxSingleStepLength = uint32_t(64) << 20; // 64MiB * Returns true when flushing is complete.
constexpr uint32_t defaultBufferLength = uint32_t(256) << 10; // 256 KiB */
bool LZMA2StreamCodec::flushVarintBuffer(MutableByteRange& output) {
if (varintToEncode_.empty()) {
return true;
}
const size_t numBytesToCopy = std::min(varintToEncode_.size(), output.size());
if (numBytesToCopy > 0) {
memcpy(output.data(), varintToEncode_.data(), numBytesToCopy);
}
varintToEncode_.advance(numBytesToCopy);
output.advance(numBytesToCopy);
return varintToEncode_.empty();
}
folly::io::Cursor cursor(data); bool LZMA2StreamCodec::doCompressStream(
ByteRange& input,
MutableByteRange& output,
StreamCodec::FlushOp flushOp) {
if (needReset_) {
resetCStream();
if (encodeSize()) { if (encodeSize()) {
const uint64_t actualUncompressedLength = decodeVarintFromCursor(cursor); varintBufferPos_ = 0;
if (uncompressedLength && *uncompressedLength != actualUncompressedLength) { size_t const varintSize =
throw std::runtime_error("LZMA2Codec: invalid uncompressed length"); encodeVarint(*uncompressedLength(), varintBuffer_.data());
varintToEncode_ = {varintBuffer_.data(), varintSize};
} }
uncompressedLength = actualUncompressedLength; needReset_ = false;
} }
auto out = addOutputBuffer( if (!flushVarintBuffer(output)) {
&stream, return false;
((uncompressedLength && *uncompressedLength <= maxSingleStepLength) }
? *uncompressedLength
: defaultBufferLength));
bool streamEnd = false; cstream_->next_in = const_cast<uint8_t*>(input.data());
auto buf = cursor.peekBytes(); cstream_->avail_in = input.size();
while (!buf.empty()) { cstream_->next_out = output.data();
stream.next_in = const_cast<uint8_t*>(buf.data()); cstream_->avail_out = output.size();
stream.avail_in = buf.size(); SCOPE_EXIT {
input.uncheckedAdvance(input.size() - cstream_->avail_in);
output.uncheckedAdvance(output.size() - cstream_->avail_out);
};
lzma_ret const rc = lzmaThrowOnError(
lzma_code(cstream_.get_pointer(), lzmaTranslateFlush(flushOp)));
switch (flushOp) {
case StreamCodec::FlushOp::NONE:
return false;
case StreamCodec::FlushOp::FLUSH:
return cstream_->avail_in == 0 && cstream_->avail_out != 0;
case StreamCodec::FlushOp::END:
return rc == LZMA_STREAM_END;
default:
throw std::invalid_argument("LZMA2StreamCodec: invalid FlushOp");
}
}
while (stream.avail_in != 0) { /**
if (streamEnd) { * Attempts to decode a varint from input.
throw std::runtime_error(to<std::string>( * The function advances input by the number of bytes read.
"LZMA2Codec: junk after end of data")); *
* If there are too many bytes and the varint is not valid, throw a
* runtime_error.
* Returns the decoded size or 0 if more bytes are needed.
*/
size_t LZMA2StreamCodec::decodeVarint(ByteRange& input) {
if (input.empty()) {
return 0;
} }
size_t const numBytesToCopy =
std::min(kMaxVarintLength64 - varintBufferPos_, input.size());
memcpy(varintBuffer_.data() + varintBufferPos_, input.data(), numBytesToCopy);
size_t const rangeSize = varintBufferPos_ + numBytesToCopy;
ByteRange range{varintBuffer_.data(), rangeSize};
auto const ret = tryDecodeVarint(range);
streamEnd = doInflate(&stream, out.get(), defaultBufferLength); if (ret.hasValue()) {
size_t const varintSize = rangeSize - range.size();
input.advance(varintSize - varintBufferPos_);
return ret.value();
} else if (ret.error() == DecodeVarintError::TooManyBytes) {
throw std::runtime_error("LZMA2StreamCodec: invalid uncompressed length");
} else {
// Too few bytes
input.advance(numBytesToCopy);
varintBufferPos_ += numBytesToCopy;
return 0;
} }
}
cursor.skip(buf.size()); bool LZMA2StreamCodec::doUncompressStream(
buf = cursor.peekBytes(); ByteRange& input,
MutableByteRange& output,
StreamCodec::FlushOp flushOp) {
if (needReset_) {
resetDStream();
needReset_ = false;
needDecodeSize_ = encodeSize();
if (encodeSize()) {
// Reset buffer
varintBufferPos_ = 0;
}
} }
while (!streamEnd) { if (needDecodeSize_) {
streamEnd = doInflate(&stream, out.get(), defaultBufferLength); // Try decoding the varint. If the input does not contain the entire varint,
// buffer the input. If the varint can not be decoded, fail.
size_t const size = decodeVarint(input);
if (!size) {
return false;
}
if (uncompressedLength() && *uncompressedLength() != size) {
throw std::runtime_error("LZMA2StreamCodec: invalid uncompressed length");
}
needDecodeSize_ = false;
} }
out->prev()->trimEnd(stream.avail_out); dstream_->next_in = const_cast<uint8_t*>(input.data());
dstream_->avail_in = input.size();
dstream_->next_out = output.data();
dstream_->avail_out = output.size();
SCOPE_EXIT {
input.advance(input.size() - dstream_->avail_in);
output.advance(output.size() - dstream_->avail_out);
};
if (uncompressedLength && *uncompressedLength != stream.total_out) { lzma_ret rc;
throw std::runtime_error( switch (flushOp) {
to<std::string>("LZMA2Codec: invalid uncompressed length")); case StreamCodec::FlushOp::NONE:
case StreamCodec::FlushOp::FLUSH:
rc = lzmaThrowOnError(lzma_code(dstream_.get_pointer(), LZMA_RUN));
break;
case StreamCodec::FlushOp::END:
rc = lzmaThrowOnError(lzma_code(dstream_.get_pointer(), LZMA_FINISH));
break;
default:
throw std::invalid_argument("LZMA2StreamCodec: invalid flush");
} }
return rc == LZMA_STREAM_END;
return out;
} }
#endif // FOLLY_HAVE_LIBLZMA #endif // FOLLY_HAVE_LIBLZMA
#ifdef FOLLY_HAVE_LIBZSTD #ifdef FOLLY_HAVE_LIBZSTD
...@@ -1945,8 +2013,8 @@ constexpr Factory ...@@ -1945,8 +2013,8 @@ constexpr Factory
#endif #endif
#if FOLLY_HAVE_LIBLZMA #if FOLLY_HAVE_LIBLZMA
{LZMA2Codec::create, nullptr}, {LZMA2StreamCodec::createCodec, LZMA2StreamCodec::createStream},
{LZMA2Codec::create, nullptr}, {LZMA2StreamCodec::createCodec, LZMA2StreamCodec::createStream},
#else #else
{}, {},
{}, {},
......
folly/io/test/CompressionTest.cpp
View file @ 9dc004f8
...@@ -175,19 +175,21 @@ static std::vector<CodecType> availableStreamCodecs() { ...@@ -175,19 +175,21 @@ static std::vector<CodecType> availableStreamCodecs() {
} }
TEST(CompressionTestNeedsUncompressedLength, Simple) { TEST(CompressionTestNeedsUncompressedLength, Simple) {
static const struct { CodecType type; bool needsUncompressedLength; } static const struct {
expectations[] = { CodecType type;
{ CodecType::NO_COMPRESSION, false }, bool needsUncompressedLength;
{ CodecType::LZ4, true }, } expectations[] = {
{ CodecType::SNAPPY, false }, {CodecType::NO_COMPRESSION, false},
{ CodecType::ZLIB, false }, {CodecType::LZ4, true},
{ CodecType::LZ4_VARINT_SIZE, false }, {CodecType::SNAPPY, false},
{ CodecType::LZMA2, false }, {CodecType::ZLIB, false},
{ CodecType::LZMA2_VARINT_SIZE, false }, {CodecType::LZ4_VARINT_SIZE, false},
{ CodecType::ZSTD, false }, {CodecType::LZMA2, false},
{ CodecType::GZIP, false }, {CodecType::LZMA2_VARINT_SIZE, false},
{ CodecType::LZ4_FRAME, false }, {CodecType::ZSTD, false},
{ CodecType::BZIP2, false }, {CodecType::GZIP, false},
{CodecType::LZ4_FRAME, false},
{CodecType::BZIP2, false},
}; };
for (auto const& test : expectations) { for (auto const& test : expectations) {
...@@ -362,6 +364,18 @@ INSTANTIATE_TEST_CASE_P( ...@@ -362,6 +364,18 @@ INSTANTIATE_TEST_CASE_P(
CodecType::LZMA2_VARINT_SIZE, CodecType::LZMA2_VARINT_SIZE,
})))); }))));
TEST(LZMATest, UncompressBadVarint) {
if (hasStreamCodec(CodecType::LZMA2_VARINT_SIZE)) {
std::string const str(kMaxVarintLength64 * 2, '\xff');
ByteRange input((folly::StringPiece(str)));
auto codec = getStreamCodec(CodecType::LZMA2_VARINT_SIZE);
auto buffer = IOBuf::create(16);
buffer->append(buffer->capacity());
MutableByteRange output{buffer->writableData(), buffer->length()};
EXPECT_THROW(codec->uncompressStream(input, output), std::runtime_error);
}
}
class CompressionCorruptionTest : public testing::TestWithParam<CodecType> { class CompressionCorruptionTest : public testing::TestWithParam<CodecType> {
protected: protected:
void SetUp() override { codec_ = getCodec(GetParam()); } void SetUp() override { codec_ = getCodec(GetParam()); }
...@@ -449,6 +463,26 @@ class StreamingUnitTest : public testing::TestWithParam<CodecType> { ...@@ -449,6 +463,26 @@ class StreamingUnitTest : public testing::TestWithParam<CodecType> {
std::unique_ptr<StreamCodec> codec_; std::unique_ptr<StreamCodec> codec_;
}; };
TEST(StreamingUnitTest, needsDataLength) {
static const struct {
CodecType type;
bool needsDataLength;
} expectations[] = {
{CodecType::ZLIB, false},
{CodecType::GZIP, false},
{CodecType::LZMA2, false},
{CodecType::LZMA2_VARINT_SIZE, true},
{CodecType::ZSTD, false},
};
for (auto const& test : expectations) {
if (hasStreamCodec(test.type)) {
EXPECT_EQ(
getStreamCodec(test.type)->needsDataLength(), test.needsDataLength);
}
}
}
TEST_P(StreamingUnitTest, maxCompressedLength) { TEST_P(StreamingUnitTest, maxCompressedLength) {
EXPECT_EQ(0, codec_->maxCompressedLength(0)); EXPECT_EQ(0, codec_->maxCompressedLength(0));
for (uint64_t const length : {1, 10, 100, 1000, 10000, 100000, 1000000}) { for (uint64_t const length : {1, 10, 100, 1000, 10000, 100000, 1000000}) {
...@@ -483,22 +517,24 @@ TEST_P(StreamingUnitTest, emptyData) { ...@@ -483,22 +517,24 @@ TEST_P(StreamingUnitTest, emptyData) {
MutableByteRange output{}; MutableByteRange output{};
// Test compressing empty data in one pass // Test compressing empty data in one pass
EXPECT_TRUE(codec_->compressStream(input, output, StreamCodec::FlushOp::END)); if (!codec_->needsDataLength()) {
EXPECT_TRUE(
codec_->compressStream(input, output, StreamCodec::FlushOp::END));
}
codec_->resetStream(0); codec_->resetStream(0);
EXPECT_TRUE(codec_->compressStream(input, output, StreamCodec::FlushOp::END)); EXPECT_TRUE(codec_->compressStream(input, output, StreamCodec::FlushOp::END));
codec_->resetStream();
output = {buffer->writableData(), buffer->length()}; output = {buffer->writableData(), buffer->length()};
EXPECT_TRUE(codec_->compressStream(input, output, StreamCodec::FlushOp::END)); EXPECT_TRUE(codec_->compressStream(input, output, StreamCodec::FlushOp::END));
EXPECT_EQ(buffer->length(), output.size()); EXPECT_EQ(buffer->length(), output.size());
// Test compressing empty data with multiple calls to compressStream() // Test compressing empty data with multiple calls to compressStream()
codec_->resetStream(); codec_->resetStream(0);
output = {}; output = {};
EXPECT_FALSE(codec_->compressStream(input, output)); EXPECT_FALSE(codec_->compressStream(input, output));
EXPECT_TRUE( EXPECT_TRUE(
codec_->compressStream(input, output, StreamCodec::FlushOp::FLUSH)); codec_->compressStream(input, output, StreamCodec::FlushOp::FLUSH));
EXPECT_TRUE(codec_->compressStream(input, output, StreamCodec::FlushOp::END)); EXPECT_TRUE(codec_->compressStream(input, output, StreamCodec::FlushOp::END));
codec_->resetStream(); codec_->resetStream(0);
output = {buffer->writableData(), buffer->length()}; output = {buffer->writableData(), buffer->length()};
EXPECT_FALSE(codec_->compressStream(input, output)); EXPECT_FALSE(codec_->compressStream(input, output));
EXPECT_TRUE( EXPECT_TRUE(
...@@ -541,7 +577,11 @@ TEST_P(StreamingUnitTest, noForwardProgressOkay) { ...@@ -541,7 +577,11 @@ TEST_P(StreamingUnitTest, noForwardProgressOkay) {
MutableByteRange emptyOutput; MutableByteRange emptyOutput;
// Compress some data to avoid empty data special casing // Compress some data to avoid empty data special casing
if (codec_->needsDataLength()) {
codec_->resetStream(inBuffer->computeChainDataLength());
} else {
codec_->resetStream(); codec_->resetStream();
}
while (!input.empty()) { while (!input.empty()) {
codec_->compressStream(input, output); codec_->compressStream(input, output);
} }
...@@ -549,7 +589,11 @@ TEST_P(StreamingUnitTest, noForwardProgressOkay) { ...@@ -549,7 +589,11 @@ TEST_P(StreamingUnitTest, noForwardProgressOkay) {
codec_->compressStream(emptyInput, emptyOutput); codec_->compressStream(emptyInput, emptyOutput);
codec_->compressStream(emptyInput, emptyOutput, StreamCodec::FlushOp::FLUSH); codec_->compressStream(emptyInput, emptyOutput, StreamCodec::FlushOp::FLUSH);
if (codec_->needsDataLength()) {
codec_->resetStream(inBuffer->computeChainDataLength());
} else {
codec_->resetStream(); codec_->resetStream();
}
input = inBuffer->coalesce(); input = inBuffer->coalesce();
output = {outBuffer->writableTail(), outBuffer->tailroom()}; output = {outBuffer->writableTail(), outBuffer->tailroom()};
while (!input.empty()) { while (!input.empty()) {
...@@ -590,6 +634,20 @@ TEST_P(StreamingUnitTest, stateTransitions) { ...@@ -590,6 +634,20 @@ TEST_P(StreamingUnitTest, stateTransitions) {
auto output = empty ? MutableByteRange{} : out; auto output = empty ? MutableByteRange{} : out;
return codec_->compressStream(input, output, flushOp); return codec_->compressStream(input, output, flushOp);
}; };
auto compress_all = [&](bool expect,
StreamCodec::FlushOp flushOp =
StreamCodec::FlushOp::NONE,
bool empty = false) {
auto input = in;
auto output = empty ? MutableByteRange{} : out;
while (!input.empty()) {
if (expect) {
EXPECT_TRUE(codec_->compressStream(input, output, flushOp));
} else {
EXPECT_FALSE(codec_->compressStream(input, output, flushOp));
}
}
};
auto uncompress = [&]( auto uncompress = [&](
StreamCodec::FlushOp flushOp = StreamCodec::FlushOp::NONE, StreamCodec::FlushOp flushOp = StreamCodec::FlushOp::NONE,
bool empty = false) { bool empty = false) {
...@@ -599,12 +657,21 @@ TEST_P(StreamingUnitTest, stateTransitions) { ...@@ -599,12 +657,21 @@ TEST_P(StreamingUnitTest, stateTransitions) {
}; };
// compression flow // compression flow
if (!codec_->needsDataLength()) {
codec_->resetStream(); codec_->resetStream();
EXPECT_FALSE(compress()); EXPECT_FALSE(compress());
EXPECT_FALSE(compress()); EXPECT_FALSE(compress());
EXPECT_TRUE(compress(StreamCodec::FlushOp::FLUSH)); EXPECT_TRUE(compress(StreamCodec::FlushOp::FLUSH));
EXPECT_FALSE(compress()); EXPECT_FALSE(compress());
EXPECT_TRUE(compress(StreamCodec::FlushOp::END)); EXPECT_TRUE(compress(StreamCodec::FlushOp::END));
}
codec_->resetStream(in.size() * 5);
compress_all(false);
compress_all(false);
compress_all(true, StreamCodec::FlushOp::FLUSH);
compress_all(false);
compress_all(true, StreamCodec::FlushOp::END);
// uncompression flow // uncompression flow
codec_->resetStream(); codec_->resetStream();
EXPECT_FALSE(uncompress(StreamCodec::FlushOp::NONE, true)); EXPECT_FALSE(uncompress(StreamCodec::FlushOp::NONE, true));
...@@ -617,34 +684,40 @@ TEST_P(StreamingUnitTest, stateTransitions) { ...@@ -617,34 +684,40 @@ TEST_P(StreamingUnitTest, stateTransitions) {
codec_->resetStream(); codec_->resetStream();
EXPECT_TRUE(uncompress(StreamCodec::FlushOp::FLUSH)); EXPECT_TRUE(uncompress(StreamCodec::FlushOp::FLUSH));
// compress -> uncompress // compress -> uncompress
codec_->resetStream(); codec_->resetStream(in.size());
EXPECT_FALSE(compress()); EXPECT_FALSE(compress());
EXPECT_THROW(uncompress(), std::logic_error); EXPECT_THROW(uncompress(), std::logic_error);
// uncompress -> compress // uncompress -> compress
codec_->resetStream(); codec_->resetStream(inBuffer->computeChainDataLength());
EXPECT_TRUE(uncompress(StreamCodec::FlushOp::FLUSH)); EXPECT_TRUE(uncompress(StreamCodec::FlushOp::FLUSH));
EXPECT_THROW(compress(), std::logic_error); EXPECT_THROW(compress(), std::logic_error);
// end -> compress // end -> compress
if (!codec_->needsDataLength()) {
codec_->resetStream(); codec_->resetStream();
EXPECT_FALSE(compress()); EXPECT_FALSE(compress());
EXPECT_TRUE(compress(StreamCodec::FlushOp::END)); EXPECT_TRUE(compress(StreamCodec::FlushOp::END));
EXPECT_THROW(compress(), std::logic_error); EXPECT_THROW(compress(), std::logic_error);
}
codec_->resetStream(in.size() * 2);
compress_all(false);
compress_all(true, StreamCodec::FlushOp::END);
EXPECT_THROW(compress(), std::logic_error);
// end -> uncompress // end -> uncompress
codec_->resetStream(); codec_->resetStream();
EXPECT_TRUE(uncompress(StreamCodec::FlushOp::FLUSH)); EXPECT_TRUE(uncompress(StreamCodec::FlushOp::FLUSH));
EXPECT_THROW(uncompress(), std::logic_error); EXPECT_THROW(uncompress(), std::logic_error);
// flush -> compress // flush -> compress
codec_->resetStream(); codec_->resetStream(in.size());
EXPECT_FALSE(compress(StreamCodec::FlushOp::FLUSH, true)); EXPECT_FALSE(compress(StreamCodec::FlushOp::FLUSH, true));
EXPECT_THROW(compress(), std::logic_error); EXPECT_THROW(compress(), std::logic_error);
// flush -> end // flush -> end
codec_->resetStream(); codec_->resetStream(in.size());
EXPECT_FALSE(compress(StreamCodec::FlushOp::FLUSH, true)); EXPECT_FALSE(compress(StreamCodec::FlushOp::FLUSH, true));
EXPECT_THROW(compress(StreamCodec::FlushOp::END), std::logic_error); EXPECT_THROW(compress(StreamCodec::FlushOp::END), std::logic_error);
// undefined -> compress // undefined -> compress
codec_->compress(inBuffer.get()); codec_->compress(inBuffer.get());
EXPECT_THROW(compress(), std::logic_error); EXPECT_THROW(compress(), std::logic_error);
codec_->uncompress(compressed.get()); codec_->uncompress(compressed.get(), inBuffer->computeChainDataLength());
EXPECT_THROW(compress(), std::logic_error); EXPECT_THROW(compress(), std::logic_error);
// undefined -> undefined // undefined -> undefined
codec_->uncompress(compressed.get()); codec_->uncompress(compressed.get());
...@@ -738,7 +811,11 @@ void StreamingCompressionTest::runResetStreamTest(DataHolder const& dh) { ...@@ -738,7 +811,11 @@ void StreamingCompressionTest::runResetStreamTest(DataHolder const& dh) {
codec_->resetStream(uncompressedLength_); codec_->resetStream(uncompressedLength_);
compressSome(codec_.get(), input, chunkSize_, StreamCodec::FlushOp::NONE); compressSome(codec_.get(), input, chunkSize_, StreamCodec::FlushOp::NONE);
// Reset stream and compress all // Reset stream and compress all
if (codec_->needsDataLength()) {
codec_->resetStream(uncompressedLength_);
} else {
codec_->resetStream(); codec_->resetStream();
}
auto compressed = auto compressed =
compressSome(codec_.get(), input, chunkSize_, StreamCodec::FlushOp::END); compressSome(codec_.get(), input, chunkSize_, StreamCodec::FlushOp::END);
auto const uncompressed = codec_->uncompress(compressed.get(), input.size()); auto const uncompressed = codec_->uncompress(compressed.get(), input.size());
...@@ -820,7 +897,11 @@ void StreamingCompressionTest::runFlushTest(DataHolder const& dh) { ...@@ -820,7 +897,11 @@ void StreamingCompressionTest::runFlushTest(DataHolder const& dh) {
auto const inputs = split(dh.data(uncompressedLength_)); auto const inputs = split(dh.data(uncompressedLength_));
auto uncodec = getStreamCodec(codec_->type()); auto uncodec = getStreamCodec(codec_->type());
if (codec_->needsDataLength()) {
codec_->resetStream(uncompressedLength_);
} else {
codec_->resetStream(); codec_->resetStream();
}
for (auto input : inputs) { for (auto input : inputs) {
// Compress some data and flush the stream // Compress some data and flush the stream
auto compressed = compressSome( auto compressed = compressSome(
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment