lib/cpp-httplib
1
0
Fork 0
mirror of synced 2025-04-20 11:47:43 +03:00
Code

Brotli suport on server. Fix #578

Browse Source
This commit is contained in:
yhirose 2020-07-31 08:22:45 -04:00
parent 3e906a9b8c
commit a5b4cfadb9
2 changed files with 219 additions and 105 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

255
httplib.h
View File

@ -225,6 +225,7 @@ inline const unsigned char *ASN1_STRING_get0_data(const ASN1_STRING *asn1) {
#ifdef CPPHTTPLIB_BROTLI_SUPPORT #ifdef CPPHTTPLIB_BROTLI_SUPPORT
#include <brotli/decode.h> #include <brotli/decode.h>
#include <brotli/encode.h>
#endif #endif
/* /*
@ -2157,8 +2158,39 @@ inline EncodingType encoding_type(const Request &req, const Response &res) {
return EncodingType::None; return EncodingType::None;
} }
class compressor {
public:
virtual ~compressor(){};
typedef std::function<bool(const char *data, size_t data_len)> Callback;
virtual bool compress(const char *data, size_t data_length, bool last,
Callback callback) = 0;
};
class decompressor {
public:
virtual ~decompressor() {}
virtual bool is_valid() const = 0;
typedef std::function<bool(const char *data, size_t data_len)> Callback;
virtual bool decompress(const char *data, size_t data_length,
Callback callback) = 0;
};
class nocompressor : public compressor {
public:
~nocompressor(){};
bool compress(const char *data, size_t data_length, bool /*last*/,
Callback callback) override {
if (!data_length) { return true; }
return callback(data, data_length);
}
};
#ifdef CPPHTTPLIB_ZLIB_SUPPORT #ifdef CPPHTTPLIB_ZLIB_SUPPORT
class gzip_compressor { class gzip_compressor : public compressor {
public: public:
gzip_compressor() { gzip_compressor() {
std::memset(&strm_, 0, sizeof(strm_)); std::memset(&strm_, 0, sizeof(strm_));
@ -2172,8 +2204,8 @@ public:
~gzip_compressor() { deflateEnd(&strm_); } ~gzip_compressor() { deflateEnd(&strm_); }
template <typename T> bool compress(const char *data, size_t data_length, bool last,
bool compress(const char *data, size_t data_length, bool last, T callback) { Callback callback) override {
assert(is_valid_); assert(is_valid_);
auto flush = last ? Z_FINISH : Z_NO_FLUSH; auto flush = last ? Z_FINISH : Z_NO_FLUSH;
@ -2206,7 +2238,7 @@ private:
z_stream strm_; z_stream strm_;
}; };
class gzip_decompressor { class gzip_decompressor : public decompressor {
public: public:
gzip_decompressor() { gzip_decompressor() {
std::memset(&strm_, 0, sizeof(strm_)); std::memset(&strm_, 0, sizeof(strm_));
@ -2223,10 +2255,10 @@ public:
~gzip_decompressor() { inflateEnd(&strm_); } ~gzip_decompressor() { inflateEnd(&strm_); }
bool is_valid() const { return is_valid_; } bool is_valid() const override { return is_valid_; }
template <typename T> bool decompress(const char *data, size_t data_length,
bool decompress(const char *data, size_t data_length, T callback) { Callback callback) override {
assert(is_valid_); assert(is_valid_);
int ret = Z_OK; int ret = Z_OK;
@ -2262,7 +2294,52 @@ private:
#endif #endif
#ifdef CPPHTTPLIB_BROTLI_SUPPORT #ifdef CPPHTTPLIB_BROTLI_SUPPORT
class brotli_decompressor { class brotli_compressor : public compressor {
public:
brotli_compressor() {
state_ = BrotliEncoderCreateInstance(nullptr, nullptr, nullptr);
}
~brotli_compressor() { BrotliEncoderDestroyInstance(state_); }
bool compress(const char *data, size_t data_length, bool last,
Callback callback) override {
std::array<uint8_t, 16384> buff{};
auto operation = last ? BROTLI_OPERATION_FINISH : BROTLI_OPERATION_PROCESS;
auto available_in = data_length;
auto next_in = reinterpret_cast<const uint8_t *>(data);
for (;;) {
if (last) {
if (BrotliEncoderIsFinished(state_)) { break; }
} else {
if (!available_in) { break; }
}
auto available_out = buff.size();
auto next_out = buff.data();
if (!BrotliEncoderCompressStream(state_, operation, &available_in,
&next_in, &available_out, &next_out,
nullptr)) {
return false;
}
auto output_bytes = buff.size() - available_out;
if (output_bytes) {
callback(reinterpret_cast<const char *>(buff.data()), output_bytes);
}
}
return true;
}
private:
BrotliEncoderState *state_ = nullptr;
};
class brotli_decompressor : public decompressor {
public: public:
brotli_decompressor() { brotli_decompressor() {
decoder_s = BrotliDecoderCreateInstance(0, 0, 0); decoder_s = BrotliDecoderCreateInstance(0, 0, 0);
@ -2274,13 +2351,14 @@ public:
if (decoder_s) { BrotliDecoderDestroyInstance(decoder_s); } if (decoder_s) { BrotliDecoderDestroyInstance(decoder_s); }
} }
bool is_valid() const { return decoder_s; } bool is_valid() const override { return decoder_s; }
template <typename T> bool decompress(const char *data, size_t data_length,
bool decompress(const char *data, size_t data_length, T callback) { Callback callback) override {
if (decoder_r == BROTLI_DECODER_RESULT_SUCCESS || if (decoder_r == BROTLI_DECODER_RESULT_SUCCESS ||
decoder_r == BROTLI_DECODER_RESULT_ERROR) decoder_r == BROTLI_DECODER_RESULT_ERROR) {
return 0; return 0;
}
const uint8_t *next_in = (const uint8_t *)data; const uint8_t *next_in = (const uint8_t *)data;
size_t avail_in = data_length; size_t avail_in = data_length;
@ -2491,32 +2569,29 @@ bool prepare_content_receiver(T &x, int &status, ContentReceiver receiver,
bool decompress, U callback) { bool decompress, U callback) {
if (decompress) { if (decompress) {
std::string encoding = x.get_header_value("Content-Encoding"); std::string encoding = x.get_header_value("Content-Encoding");
std::shared_ptr<decompressor> decompressor;
if (encoding.find("gzip") != std::string::npos || if (encoding.find("gzip") != std::string::npos ||
encoding.find("deflate") != std::string::npos) { encoding.find("deflate") != std::string::npos) {
#ifdef CPPHTTPLIB_ZLIB_SUPPORT #ifdef CPPHTTPLIB_ZLIB_SUPPORT
gzip_decompressor decompressor; decompressor = std::make_shared<gzip_decompressor>();
if (decompressor.is_valid()) {
ContentReceiver out = [&](const char *buf, size_t n) {
return decompressor.decompress(
buf, n,
[&](const char *buf, size_t n) { return receiver(buf, n); });
};
return callback(out);
} else {
status = 500;
return false;
}
#else #else
status = 415; status = 415;
return false; return false;
#endif #endif
} else if (encoding.find("br") != std::string::npos) { } else if (encoding.find("br") != std::string::npos) {
#ifdef CPPHTTPLIB_BROTLI_SUPPORT #ifdef CPPHTTPLIB_BROTLI_SUPPORT
brotli_decompressor decompressor; decompressor = std::make_shared<brotli_decompressor>();
if (decompressor.is_valid()) { #else
status = 415;
return false;
#endif
}
if (decompressor) {
if (decompressor->is_valid()) {
ContentReceiver out = [&](const char *buf, size_t n) { ContentReceiver out = [&](const char *buf, size_t n) {
return decompressor.decompress( return decompressor->decompress(
buf, n, buf, n,
[&](const char *buf, size_t n) { return receiver(buf, n); }); [&](const char *buf, size_t n) { return receiver(buf, n); });
}; };
@ -2525,17 +2600,12 @@ bool prepare_content_receiver(T &x, int &status, ContentReceiver receiver,
status = 500; status = 500;
return false; return false;
} }
#else
status = 415;
return false;
#endif
} }
} }
ContentReceiver out = [&](const char *buf, size_t n) { ContentReceiver out = [&](const char *buf, size_t n) {
return receiver(buf, n); return receiver(buf, n);
}; };
return callback(out); return callback(out);
} }
@ -2628,10 +2698,10 @@ inline ssize_t write_content(Stream &strm, ContentProvider content_provider,
return static_cast<ssize_t>(offset - begin_offset); return static_cast<ssize_t>(offset - begin_offset);
} }
template <typename T> template <typename T, typename U>
inline ssize_t write_content_chunked(Stream &strm, inline ssize_t write_content_chunked(Stream &strm,
ContentProvider content_provider, ContentProvider content_provider,
T is_shutting_down, EncodingType type) { T is_shutting_down, U &compressor) {
size_t offset = 0; size_t offset = 0;
auto data_available = true; auto data_available = true;
ssize_t total_written_length = 0; ssize_t total_written_length = 0;
@ -2639,10 +2709,6 @@ inline ssize_t write_content_chunked(Stream &strm,
auto ok = true; auto ok = true;
DataSink data_sink; DataSink data_sink;
#ifdef CPPHTTPLIB_ZLIB_SUPPORT
detail::gzip_compressor compressor;
#endif
data_sink.write = [&](const char *d, size_t l) { data_sink.write = [&](const char *d, size_t l) {
if (!ok) { return; } if (!ok) { return; }
@ -2650,22 +2716,13 @@ inline ssize_t write_content_chunked(Stream &strm,
offset += l; offset += l;
std::string payload; std::string payload;
if (type == EncodingType::Gzip) { if (!compressor.compress(d, l, false,
#ifdef CPPHTTPLIB_ZLIB_SUPPORT [&](const char *data, size_t data_len) {
if (!compressor.compress(d, l, false, payload.append(data, data_len);
[&](const char *data, size_t data_len) { return true;
payload.append(data, data_len); })) {
return true; ok = false;
})) { return;
ok = false;
return;
}
#endif
} else if (type == EncodingType::Brotli) {
#ifdef CPPHTTPLIB_BROTLI_SUPPORT
#endif
} else {
payload = std::string(d, l);
} }
if (!payload.empty()) { if (!payload.empty()) {
@ -2685,32 +2742,25 @@ inline ssize_t write_content_chunked(Stream &strm,
data_available = false; data_available = false;
if (type == EncodingType::Gzip) { std::string payload;
#ifdef CPPHTTPLIB_ZLIB_SUPPORT if (!compressor.compress(nullptr, 0, true,
std::string payload; [&](const char *data, size_t data_len) {
if (!compressor.compress(nullptr, 0, true, payload.append(data, data_len);
[&](const char *data, size_t data_len) { return true;
payload.append(data, data_len); })) {
return true; ok = false;
})) { return;
}
if (!payload.empty()) {
// Emit chunked response header and footer for each chunk
auto chunk = from_i_to_hex(payload.size()) + "\r\n" + payload + "\r\n";
if (write_data(strm, chunk.data(), chunk.size())) {
total_written_length += chunk.size();
} else {
ok = false; ok = false;
return; return;
} }
if (!payload.empty()) {
// Emit chunked response header and footer for each chunk
auto chunk = from_i_to_hex(payload.size()) + "\r\n" + payload + "\r\n";
if (write_data(strm, chunk.data(), chunk.size())) {
total_written_length += chunk.size();
} else {
ok = false;
return;
}
}
#endif
} else if (type == EncodingType::Brotli) {
#ifdef CPPHTTPLIB_BROTLI_SUPPORT
#endif
} }
static const std::string done_marker("0\r\n\r\n"); static const std::string done_marker("0\r\n\r\n");
@ -3918,25 +3968,33 @@ inline bool Server::write_response(Stream &strm, bool close_connection,
} }
if (type != detail::EncodingType::None) { if (type != detail::EncodingType::None) {
#ifdef CPPHTTPLIB_ZLIB_SUPPORT std::shared_ptr<detail::compressor> compressor;
std::string compressed;
if (type == detail::EncodingType::Gzip) { if (type == detail::EncodingType::Gzip) {
detail::gzip_compressor compressor; #ifdef CPPHTTPLIB_ZLIB_SUPPORT
if (!compressor.compress(res.body.data(), res.body.size(), true, compressor = std::make_shared<detail::gzip_compressor>();
[&](const char *data, size_t data_len) {
compressed.append(data, data_len);
return true;
})) {
return false;
}
res.set_header("Content-Encoding", "gzip"); res.set_header("Content-Encoding", "gzip");
#endif
} else if (type == detail::EncodingType::Brotli) { } else if (type == detail::EncodingType::Brotli) {
// TODO: #ifdef CPPHTTPLIB_BROTLI_SUPPORT
compressor = std::make_shared<detail::brotli_compressor>();
res.set_header("Content-Encoding", "brotli");
#endif
} }
res.body.swap(compressed); if (compressor) {
#endif std::string compressed;
if (!compressor->compress(res.body.data(), res.body.size(), true,
[&](const char *data, size_t data_len) {
compressed.append(data, data_len);
return true;
})) {
return false;
}
res.body.swap(compressed);
}
} }
auto length = std::to_string(res.body.size()); auto length = std::to_string(res.body.size());
@ -3999,8 +4057,23 @@ Server::write_content_with_provider(Stream &strm, const Request &req,
} }
} else { } else {
auto type = detail::encoding_type(req, res); auto type = detail::encoding_type(req, res);
std::shared_ptr<detail::compressor> compressor;
if (type == detail::EncodingType::Gzip) {
#ifdef CPPHTTPLIB_ZLIB_SUPPORT
compressor = std::make_shared<detail::gzip_compressor>();
#endif
} else if (type == detail::EncodingType::Brotli) {
#ifdef CPPHTTPLIB_BROTLI_SUPPORT
compressor = std::make_shared<detail::brotli_compressor>();
#endif
} else {
compressor = std::make_shared<detail::nocompressor>();
}
assert(compressor != nullptr);
if (detail::write_content_chunked(strm, res.content_provider_, if (detail::write_content_chunked(strm, res.content_provider_,
is_shutting_down, type) < 0) { is_shutting_down, *compressor) < 0) {
return false; return false;
} }
} }

69
test/test.cc
View File

@ -1227,22 +1227,22 @@ protected:
[&](const Request &req, Response & /*res*/) { [&](const Request &req, Response & /*res*/) {
EXPECT_EQ("close", req.get_header_value("Connection")); EXPECT_EQ("close", req.get_header_value("Connection"));
}) })
#ifdef CPPHTTPLIB_ZLIB_SUPPORT #if defined(CPPHTTPLIB_ZLIB_SUPPORT) || defined(CPPHTTPLIB_BROTLI_SUPPORT)
.Get("/gzip", .Get("/compress",
[&](const Request & /*req*/, Response &res) { [&](const Request & /*req*/, Response &res) {
res.set_content( res.set_content(
"12345678901234567890123456789012345678901234567890123456789" "12345678901234567890123456789012345678901234567890123456789"
"01234567890123456789012345678901234567890", "01234567890123456789012345678901234567890",
"text/plain"); "text/plain");
}) })
.Get("/nogzip", .Get("/nocompress",
[&](const Request & /*req*/, Response &res) { [&](const Request & /*req*/, Response &res) {
res.set_content( res.set_content(
"12345678901234567890123456789012345678901234567890123456789" "12345678901234567890123456789012345678901234567890123456789"
"01234567890123456789012345678901234567890", "01234567890123456789012345678901234567890",
"application/octet-stream"); "application/octet-stream");
}) })
.Post("/gzipmultipart", .Post("/compress-multipart",
[&](const Request &req, Response & /*res*/) { [&](const Request &req, Response & /*res*/) {
EXPECT_EQ(2u, req.files.size()); EXPECT_EQ(2u, req.files.size());
ASSERT_TRUE(!req.has_file("???")); ASSERT_TRUE(!req.has_file("???"));
@ -2123,6 +2123,28 @@ TEST_F(ServerTest, GetStreamedChunkedWithGzip2) {
} }
#endif #endif
#ifdef CPPHTTPLIB_BROTLI_SUPPORT
TEST_F(ServerTest, GetStreamedChunkedWithBrotli) {
httplib::Headers headers;
headers.emplace("Accept-Encoding", "brotli");
auto res = cli_.Get("/streamed-chunked", headers);
ASSERT_TRUE(res != nullptr);
EXPECT_EQ(200, res->status);
EXPECT_EQ(std::string("123456789"), res->body);
}
TEST_F(ServerTest, GetStreamedChunkedWithBrotli2) {
httplib::Headers headers;
headers.emplace("Accept-Encoding", "brotli");
auto res = cli_.Get("/streamed-chunked2", headers);
ASSERT_TRUE(res != nullptr);
EXPECT_EQ(200, res->status);
EXPECT_EQ(std::string("123456789"), res->body);
}
#endif
TEST_F(ServerTest, Patch) { TEST_F(ServerTest, Patch) {
auto res = cli_.Patch("/patch", "PATCH", "text/plain"); auto res = cli_.Patch("/patch", "PATCH", "text/plain");
ASSERT_TRUE(res != nullptr); ASSERT_TRUE(res != nullptr);
@ -2285,7 +2307,7 @@ TEST_F(ServerTest, KeepAlive) {
TEST_F(ServerTest, Gzip) { TEST_F(ServerTest, Gzip) {
Headers headers; Headers headers;
headers.emplace("Accept-Encoding", "gzip, deflate"); headers.emplace("Accept-Encoding", "gzip, deflate");
auto res = cli_.Get("/gzip", headers); auto res = cli_.Get("/compress", headers);
ASSERT_TRUE(res != nullptr); ASSERT_TRUE(res != nullptr);
EXPECT_EQ("gzip", res->get_header_value("Content-Encoding")); EXPECT_EQ("gzip", res->get_header_value("Content-Encoding"));
@ -2299,7 +2321,7 @@ TEST_F(ServerTest, Gzip) {
TEST_F(ServerTest, GzipWithoutAcceptEncoding) { TEST_F(ServerTest, GzipWithoutAcceptEncoding) {
Headers headers; Headers headers;
auto res = cli_.Get("/gzip", headers); auto res = cli_.Get("/compress", headers);
ASSERT_TRUE(res != nullptr); ASSERT_TRUE(res != nullptr);
EXPECT_TRUE(res->get_header_value("Content-Encoding").empty()); EXPECT_TRUE(res->get_header_value("Content-Encoding").empty());
@ -2316,7 +2338,7 @@ TEST_F(ServerTest, GzipWithContentReceiver) {
headers.emplace("Accept-Encoding", "gzip, deflate"); headers.emplace("Accept-Encoding", "gzip, deflate");
std::string body; std::string body;
auto res = auto res =
cli_.Get("/gzip", headers, [&](const char *data, uint64_t data_length) { cli_.Get("/compress", headers, [&](const char *data, uint64_t data_length) {
EXPECT_EQ(data_length, 100); EXPECT_EQ(data_length, 100);
body.append(data, data_length); body.append(data, data_length);
return true; return true;
@ -2337,7 +2359,7 @@ TEST_F(ServerTest, GzipWithoutDecompressing) {
headers.emplace("Accept-Encoding", "gzip, deflate"); headers.emplace("Accept-Encoding", "gzip, deflate");
cli_.set_decompress(false); cli_.set_decompress(false);
auto res = cli_.Get("/gzip", headers); auto res = cli_.Get("/compress", headers);
ASSERT_TRUE(res != nullptr); ASSERT_TRUE(res != nullptr);
EXPECT_EQ("gzip", res->get_header_value("Content-Encoding")); EXPECT_EQ("gzip", res->get_header_value("Content-Encoding"));
@ -2351,7 +2373,7 @@ TEST_F(ServerTest, GzipWithContentReceiverWithoutAcceptEncoding) {
Headers headers; Headers headers;
std::string body; std::string body;
auto res = auto res =
cli_.Get("/gzip", headers, [&](const char *data, uint64_t data_length) { cli_.Get("/compress", headers, [&](const char *data, uint64_t data_length) {
EXPECT_EQ(data_length, 100); EXPECT_EQ(data_length, 100);
body.append(data, data_length); body.append(data, data_length);
return true; return true;
@ -2370,7 +2392,7 @@ TEST_F(ServerTest, GzipWithContentReceiverWithoutAcceptEncoding) {
TEST_F(ServerTest, NoGzip) { TEST_F(ServerTest, NoGzip) {
Headers headers; Headers headers;
headers.emplace("Accept-Encoding", "gzip, deflate"); headers.emplace("Accept-Encoding", "gzip, deflate");
auto res = cli_.Get("/nogzip", headers); auto res = cli_.Get("/nocompress", headers);
ASSERT_TRUE(res != nullptr); ASSERT_TRUE(res != nullptr);
EXPECT_EQ(false, res->has_header("Content-Encoding")); EXPECT_EQ(false, res->has_header("Content-Encoding"));
@ -2387,7 +2409,7 @@ TEST_F(ServerTest, NoGzipWithContentReceiver) {
headers.emplace("Accept-Encoding", "gzip, deflate"); headers.emplace("Accept-Encoding", "gzip, deflate");
std::string body; std::string body;
auto res = auto res =
cli_.Get("/nogzip", headers, [&](const char *data, uint64_t data_length) { cli_.Get("/nocompress", headers, [&](const char *data, uint64_t data_length) {
EXPECT_EQ(data_length, 100); EXPECT_EQ(data_length, 100);
body.append(data, data_length); body.append(data, data_length);
return true; return true;
@ -2410,13 +2432,30 @@ TEST_F(ServerTest, MultipartFormDataGzip) {
}; };
cli_.set_compress(true); cli_.set_compress(true);
auto res = cli_.Post("/gzipmultipart", items); auto res = cli_.Post("/compress-multipart", items);
ASSERT_TRUE(res != nullptr); ASSERT_TRUE(res != nullptr);
EXPECT_EQ(200, res->status); EXPECT_EQ(200, res->status);
} }
#endif #endif
#ifdef CPPHTTPLIB_BROTLI_SUPPORT
TEST_F(ServerTest, Brotli) {
Headers headers;
headers.emplace("Accept-Encoding", "br");
auto res = cli_.Get("/compress", headers);
ASSERT_TRUE(res != nullptr);
EXPECT_EQ("brotli", res->get_header_value("Content-Encoding"));
EXPECT_EQ("text/plain", res->get_header_value("Content-Type"));
EXPECT_EQ("19", res->get_header_value("Content-Length"));
EXPECT_EQ("123456789012345678901234567890123456789012345678901234567890123456"
"7890123456789012345678901234567890",
res->body);
EXPECT_EQ(200, res->status);
}
#endif
// Sends a raw request to a server listening at HOST:PORT. // Sends a raw request to a server listening at HOST:PORT.
static bool send_request(time_t read_timeout_sec, const std::string &req, static bool send_request(time_t read_timeout_sec, const std::string &req,
std::string *resp = nullptr) { std::string *resp = nullptr) {
@ -3149,12 +3188,14 @@ TEST(YahooRedirectTest3, SimpleInterface) {
#ifdef CPPHTTPLIB_BROTLI_SUPPORT #ifdef CPPHTTPLIB_BROTLI_SUPPORT
TEST(DecodeWithChunkedEncoding, BrotliEncoding) { TEST(DecodeWithChunkedEncoding, BrotliEncoding) {
httplib::Client cli("https://cdnjs.cloudflare.com"); httplib::Client cli("https://cdnjs.cloudflare.com");
auto res = cli.Get("/ajax/libs/jquery/3.5.1/jquery.js", {{"Accept-Encoding", "brotli"}}); auto res = cli.Get("/ajax/libs/jquery/3.5.1/jquery.js",
{{"Accept-Encoding", "brotli"}});
ASSERT_TRUE(res != nullptr); ASSERT_TRUE(res != nullptr);
EXPECT_EQ(200, res->status); EXPECT_EQ(200, res->status);
EXPECT_EQ(287630, res->body.size()); EXPECT_EQ(287630, res->body.size());
EXPECT_EQ("application/javascript; charset=utf-8", res->get_header_value("Content-Type")); EXPECT_EQ("application/javascript; charset=utf-8",
res->get_header_value("Content-Type"));
} }
#endif #endif