lib/cpp-httplib
1
0
Fork 0
mirror of synced 2025-04-20 11:47:43 +03:00
Code

Fix gzip compression/decompression over 4 GiB data size (#1002)

Browse Source 
* Fix gzip compression/decompression over 4 GiB data size

* Add gzip test for large random data
This commit is contained in:
yosh-matsuda 2021-07-23 11:07:40 +09:00 committed by GitHub
parent 52f5eb5980
commit 879dd261c2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 106 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
httplib.h
View File

@ -2578,11 +2578,19 @@ public:
Callback callback) override { Callback callback) override {
assert(is_valid_); assert(is_valid_);
auto flush = last ? Z_FINISH : Z_NO_FLUSH; do {
constexpr size_t max_avail_in =
std::numeric_limits<decltype(strm_.avail_in)>::max();
strm_.avail_in = static_cast<decltype(strm_.avail_in)>(data_length); strm_.avail_in = static_cast<decltype(strm_.avail_in)>(
strm_.next_in = const_cast<Bytef *>(reinterpret_cast<const Bytef *>(data)); std::min(data_length, max_avail_in));
strm_.next_in =
const_cast<Bytef *>(reinterpret_cast<const Bytef *>(data));
data_length -= strm_.avail_in;
data += strm_.avail_in;
auto flush = (last && data_length == 0) ? Z_FINISH : Z_NO_FLUSH;
int ret = Z_OK; int ret = Z_OK;
std::array<char, CPPHTTPLIB_COMPRESSION_BUFSIZ> buff{}; std::array<char, CPPHTTPLIB_COMPRESSION_BUFSIZ> buff{};
@ -2598,8 +2606,12 @@ public:
} }
} while (strm_.avail_out == 0); } while (strm_.avail_out == 0);
assert((last && ret == Z_STREAM_END) || (!last && ret == Z_OK)); assert((flush == Z_FINISH && ret == Z_STREAM_END) ||
(flush == Z_NO_FLUSH && ret == Z_OK));
assert(strm_.avail_in == 0); assert(strm_.avail_in == 0);
} while (data_length > 0);
return true; return true;
} }
@ -2633,8 +2645,17 @@ public:
int ret = Z_OK; int ret = Z_OK;
strm_.avail_in = static_cast<decltype(strm_.avail_in)>(data_length); do {
strm_.next_in = const_cast<Bytef *>(reinterpret_cast<const Bytef *>(data)); constexpr size_t max_avail_in =
std::numeric_limits<decltype(strm_.avail_in)>::max();
strm_.avail_in = static_cast<decltype(strm_.avail_in)>(
std::min(data_length, max_avail_in));
strm_.next_in =
const_cast<Bytef *>(reinterpret_cast<const Bytef *>(data));
data_length -= strm_.avail_in;
data += strm_.avail_in;
std::array<char, CPPHTTPLIB_COMPRESSION_BUFSIZ> buff{}; std::array<char, CPPHTTPLIB_COMPRESSION_BUFSIZ> buff{};
while (strm_.avail_in > 0) { while (strm_.avail_in > 0) {
@ -2654,7 +2675,11 @@ public:
} }
} }
return ret == Z_OK || ret == Z_STREAM_END; if (ret != Z_OK && ret != Z_STREAM_END) return false;
} while (data_length > 0);
return true;
} }
private: private:

46
test/test.cc
View File

@ -2868,6 +2868,52 @@ TEST(GzipDecompressor, ChunkedDecompression) {
} }
ASSERT_EQ(data, decompressed_data); ASSERT_EQ(data, decompressed_data);
} }
TEST(GzipDecompressor, LargeRandomData) {
// prepare large random data that is difficult to be compressed and is
// expected to have large size even when compressed
std::random_device seed_gen;
std::mt19937 random(seed_gen());
constexpr auto large_size_byte = 4294967296UL; // 4GiB
constexpr auto data_size = large_size_byte + 134217728UL; // + 128MiB
std::vector<std::uint32_t> data(data_size / sizeof(std::uint32_t));
std::generate(data.begin(), data.end(), [&]() { return random(); });
// compress data over 4GiB
std::string compressed_data;
compressed_data.reserve(large_size_byte + 536870912UL); // + 512MiB reserved
httplib::detail::gzip_compressor compressor;
auto result = compressor.compress(reinterpret_cast<const char *>(data.data()),
data.size() * sizeof(std::uint32_t), true,
[&](const char *data, size_t size) {
compressed_data.insert(
compressed_data.size(), data, size);
return true;
});
ASSERT_TRUE(result);
// FIXME: compressed data size is expected to be greater than 4GiB,
// but there is no guarantee
// ASSERT_TRUE(compressed_data.size() >= large_size_byte);
// decompress data over 4GiB
std::string decompressed_data;
decompressed_data.reserve(data_size);
httplib::detail::gzip_decompressor decompressor;
result = decompressor.decompress(
compressed_data.data(), compressed_data.size(),
[&](const char *data, size_t size) {
decompressed_data.insert(decompressed_data.size(), data, size);
return true;
});
ASSERT_TRUE(result);
// compare
ASSERT_EQ(data_size, decompressed_data.size());
ASSERT_TRUE(std::memcmp(data.data(), decompressed_data.data(), data_size) ==
0);
}
#endif #endif
#ifdef CPPHTTPLIB_BROTLI_SUPPORT #ifdef CPPHTTPLIB_BROTLI_SUPPORT