Add test of httplib.h split into .h + .cc (#1015)

In order to test the split version (.h + .cc via split.py):

- Added a test_split program in the test directory whose main purpose is
  to verify that it works to compile and link the test case code against
  the split httplib.h version.
- Moved types needed for test cases to the “header part” of httplib.h.
  Also added forward declarations of functions needed by test cases.
- Added an include_httplib.cc file which is linked together with test.cc
  to verify that inline keywords have not been forgotten.

The changes to httplib.h just move code around (or add forward
declarations), with one exception: detail::split and
detail::process_client_socket have been converted to non-template
functions (taking an std::function instead of using a type parameter for
the function) and forward-declared instead. This avoids having to move
the templates to the “header part”.

.github/workflows/test.yaml

@@ -26,7 +26,7 @@ jobs:
       run: brew install brotli
     - name: make
       if: matrix.os != 'windows-latest'
-      run: cd test && make
+      run: cd test && make -j2
     - name: check fuzz test target
       if: matrix.os == 'ubuntu-latest'
       run: cd test && make -f Makefile.fuzz_test

.gitignore

@@ -10,8 +10,11 @@ example/redirect
 example/sse*
 example/upload
 example/*.pem
+test/httplib.cc
+test/httplib.h
 test/test
 test/test_proxy
+test/test_split
 test/test.xcodeproj/xcuser*
 test/test.xcodeproj/*/xcuser*
 test/*.pem

httplib.h

@@ -1613,10 +1613,191 @@ Client::set_write_timeout(const std::chrono::duration<Rep, Period> &duration) {
   cli_->set_write_timeout(duration);
 }
 
+/*
+ * Forward declarations and types that will be part of the .h file if split into
+ * .h + .cc.
+ */
+
+std::pair<std::string, std::string> make_range_header(Ranges ranges);
+
+std::pair<std::string, std::string>
+make_basic_authentication_header(const std::string &username,
+                                 const std::string &password,
+                                 bool is_proxy = false);
+
+namespace detail {
+
+std::string encode_query_param(const std::string &value);
+
+void read_file(const std::string &path, std::string &out);
+
+std::string trim_copy(const std::string &s);
+
+void split(const char *b, const char *e, char d,
+           std::function<void(const char *, const char *)> fn);
+
+bool process_client_socket(socket_t sock, time_t read_timeout_sec,
+                           time_t read_timeout_usec, time_t write_timeout_sec,
+                           time_t write_timeout_usec,
+                           std::function<bool(Stream &)> callback);
+
+socket_t create_client_socket(const char *host, int port, int address_family,
+                              bool tcp_nodelay, SocketOptions socket_options,
+                              time_t connection_timeout_sec,
+                              time_t connection_timeout_usec,
+                              time_t read_timeout_sec, time_t read_timeout_usec,
+                              time_t write_timeout_sec,
+                              time_t write_timeout_usec,
+                              const std::string &intf, Error &error);
+
+const char *get_header_value(const Headers &headers, const char *key,
+                             size_t id = 0, const char *def = nullptr);
+
+std::string params_to_query_str(const Params &params);
+
+void parse_query_text(const std::string &s, Params &params);
+
+bool parse_range_header(const std::string &s, Ranges &ranges);
+
+int close_socket(socket_t sock);
+
+enum class EncodingType { None = 0, Gzip, Brotli };
+
+EncodingType encoding_type(const Request &req, const Response &res);
+
+class BufferStream : public Stream {
+public:
+  BufferStream() = default;
+  ~BufferStream() override = default;
+
+  bool is_readable() const override;
+  bool is_writable() const override;
+  ssize_t read(char *ptr, size_t size) override;
+  ssize_t write(const char *ptr, size_t size) override;
+  void get_remote_ip_and_port(std::string &ip, int &port) const override;
+  socket_t socket() const override;
+
+  const std::string &get_buffer() const;
+
+private:
+  std::string buffer;
+  size_t position = 0;
+};
+
+class compressor {
+public:
+  virtual ~compressor() = default;
+
+  typedef std::function<bool(const char *data, size_t data_len)> Callback;
+  virtual bool compress(const char *data, size_t data_length, bool last,
+                        Callback callback) = 0;
+};
+
+class decompressor {
+public:
+  virtual ~decompressor() = default;
+
+  virtual bool is_valid() const = 0;
+
+  typedef std::function<bool(const char *data, size_t data_len)> Callback;
+  virtual bool decompress(const char *data, size_t data_length,
+                          Callback callback) = 0;
+};
+
+class nocompressor : public compressor {
+public:
+  virtual ~nocompressor() = default;
+
+  bool compress(const char *data, size_t data_length, bool /*last*/,
+                Callback callback) override;
+};
+
+#ifdef CPPHTTPLIB_ZLIB_SUPPORT
+class gzip_compressor : public compressor {
+public:
+  gzip_compressor();
+  ~gzip_compressor();
+
+  bool compress(const char *data, size_t data_length, bool last,
+                Callback callback) override;
+
+private:
+  bool is_valid_ = false;
+  z_stream strm_;
+};
+
+class gzip_decompressor : public decompressor {
+public:
+  gzip_decompressor();
+  ~gzip_decompressor();
+
+  bool is_valid() const override;
+
+  bool decompress(const char *data, size_t data_length,
+                  Callback callback) override;
+
+private:
+  bool is_valid_ = false;
+  z_stream strm_;
+};
+#endif
+
+#ifdef CPPHTTPLIB_BROTLI_SUPPORT
+class brotli_compressor : public compressor {
+public:
+  brotli_compressor();
+  ~brotli_compressor();
+
+  bool compress(const char *data, size_t data_length, bool last,
+                Callback callback) override;
+
+private:
+  BrotliEncoderState *state_ = nullptr;
+};
+
+class brotli_decompressor : public decompressor {
+public:
+  brotli_decompressor();
+  ~brotli_decompressor();
+
+  bool is_valid() const override;
+
+  bool decompress(const char *data, size_t data_length,
+                  Callback callback) override;
+
+private:
+  BrotliDecoderResult decoder_r;
+  BrotliDecoderState *decoder_s = nullptr;
+};
+#endif
+
+// NOTE: until the read size reaches `fixed_buffer_size`, use `fixed_buffer`
+// to store data. The call can set memory on stack for performance.
+class stream_line_reader {
+public:
+  stream_line_reader(Stream &strm, char *fixed_buffer,
+                     size_t fixed_buffer_size);
+  const char *ptr() const;
+  size_t size() const;
+  bool end_with_crlf() const;
+  bool getline();
+
+private:
+  void append(char c);
+
+  Stream &strm_;
+  char *fixed_buffer_;
+  const size_t fixed_buffer_size_;
+  size_t fixed_buffer_used_size_ = 0;
+  std::string glowable_buffer_;
+};
+
+} // namespace detail
+
 // ----------------------------------------------------------------------------
 
 /*
- * Implementation
+ * Implementation that will be part of the .cc file if split into .h + .cc.
  */
 
 namespace detail {
@@ -1895,7 +2076,8 @@ inline std::string trim_copy(const std::string &s) {
   return s.substr(r.first, r.second - r.first);
 }
 
-template <class Fn> void split(const char *b, const char *e, char d, Fn fn) {
+inline void split(const char *b, const char *e, char d,
+                  std::function<void(const char *, const char *)> fn) {
   size_t i = 0;
   size_t beg = 0;
 
@@ -1914,15 +2096,12 @@ template <class Fn> void split(const char *b, const char *e, char d, Fn fn) {
   }
 }
 
-// NOTE: until the read size reaches `fixed_buffer_size`, use `fixed_buffer`
+inline stream_line_reader::stream_line_reader(Stream &strm, char *fixed_buffer,
-// to store data. The call can set memory on stack for performance.
+                                              size_t fixed_buffer_size)
-class stream_line_reader {
-public:
-  stream_line_reader(Stream &strm, char *fixed_buffer, size_t fixed_buffer_size)
     : strm_(strm), fixed_buffer_(fixed_buffer),
       fixed_buffer_size_(fixed_buffer_size) {}
 
-  const char *ptr() const {
+inline const char *stream_line_reader::ptr() const {
   if (glowable_buffer_.empty()) {
     return fixed_buffer_;
   } else {
@@ -1930,7 +2109,7 @@ public:
   }
 }
 
-  size_t size() const {
+inline size_t stream_line_reader::size() const {
   if (glowable_buffer_.empty()) {
     return fixed_buffer_used_size_;
   } else {
@@ -1938,12 +2117,12 @@ public:
   }
 }
 
-  bool end_with_crlf() const {
+inline bool stream_line_reader::end_with_crlf() const {
   auto end = ptr() + size();
   return size() >= 2 && end[-2] == '\r' && end[-1] == '\n';
 }
 
-  bool getline() {
+inline bool stream_line_reader::getline() {
   fixed_buffer_used_size_ = 0;
   glowable_buffer_.clear();
 
@@ -1969,8 +2148,7 @@ public:
   return true;
 }
 
-private:
+inline void stream_line_reader::append(char c) {
-  void append(char c) {
   if (fixed_buffer_used_size_ < fixed_buffer_size_ - 1) {
     fixed_buffer_[fixed_buffer_used_size_++] = c;
     fixed_buffer_[fixed_buffer_used_size_] = '\0';
@@ -1983,13 +2161,6 @@ private:
   }
 }
 
-  Stream &strm_;
-  char *fixed_buffer_;
-  const size_t fixed_buffer_size_;
-  size_t fixed_buffer_used_size_ = 0;
-  std::string glowable_buffer_;
-};
-
 inline int close_socket(socket_t sock) {
 #ifdef _WIN32
   return closesocket(sock);
@@ -2159,25 +2330,6 @@ private:
 };
 #endif
 
-class BufferStream : public Stream {
-public:
-  BufferStream() = default;
-  ~BufferStream() override = default;
-
-  bool is_readable() const override;
-  bool is_writable() const override;
-  ssize_t read(char *ptr, size_t size) override;
-  ssize_t write(const char *ptr, size_t size) override;
-  void get_remote_ip_and_port(std::string &ip, int &port) const override;
-  socket_t socket() const override;
-
-  const std::string &get_buffer() const;
-
-private:
-  std::string buffer;
-  size_t position = 0;
-};
-
 inline bool keep_alive(socket_t sock, time_t keep_alive_timeout_sec) {
   using namespace std::chrono;
   auto start = steady_clock::now();
@@ -2229,11 +2381,11 @@ process_server_socket(socket_t sock, size_t keep_alive_max_count,
       });
 }
 
-template <typename T>
 inline bool process_client_socket(socket_t sock, time_t read_timeout_sec,
                                   time_t read_timeout_usec,
                                   time_t write_timeout_sec,
-                                  time_t write_timeout_usec, T callback) {
+                                  time_t write_timeout_usec,
+                                  std::function<bool(Stream &)> callback) {
   SocketStream strm(sock, read_timeout_sec, read_timeout_usec,
                     write_timeout_sec, write_timeout_usec);
   return callback(strm);
@@ -2650,8 +2802,6 @@ inline bool can_compress_content_type(const std::string &content_type) {
          content_type == "application/xhtml+xml";
 }
 
-enum class EncodingType { None = 0, Gzip, Brotli };
-
 inline EncodingType encoding_type(const Request &req, const Response &res) {
   auto ret =
       detail::can_compress_content_type(res.get_header_value("Content-Type"));
@@ -2675,41 +2825,14 @@ inline EncodingType encoding_type(const Request &req, const Response &res) {
   return EncodingType::None;
 }
 
-class compressor {
+inline bool nocompressor::compress(const char *data, size_t data_length,
-public:
+                                   bool /*last*/, Callback callback) {
-  virtual ~compressor(){};
-
-  typedef std::function<bool(const char *data, size_t data_len)> Callback;
-  virtual bool compress(const char *data, size_t data_length, bool last,
-                        Callback callback) = 0;
-};
-
-class decompressor {
-public:
-  virtual ~decompressor() {}
-
-  virtual bool is_valid() const = 0;
-
-  typedef std::function<bool(const char *data, size_t data_len)> Callback;
-  virtual bool decompress(const char *data, size_t data_length,
-                          Callback callback) = 0;
-};
-
-class nocompressor : public compressor {
-public:
-  ~nocompressor(){};
-
-  bool compress(const char *data, size_t data_length, bool /*last*/,
-                Callback callback) override {
   if (!data_length) { return true; }
   return callback(data, data_length);
 }
-};
 
 #ifdef CPPHTTPLIB_ZLIB_SUPPORT
-class gzip_compressor : public compressor {
+inline gzip_compressor::gzip_compressor() {
-public:
-  gzip_compressor() {
   std::memset(&strm_, 0, sizeof(strm_));
   strm_.zalloc = Z_NULL;
   strm_.zfree = Z_NULL;
@@ -2719,10 +2842,10 @@ public:
                            Z_DEFAULT_STRATEGY) == Z_OK;
 }
 
-  ~gzip_compressor() { deflateEnd(&strm_); }
+inline gzip_compressor::~gzip_compressor() { deflateEnd(&strm_); }
 
-  bool compress(const char *data, size_t data_length, bool last,
+inline bool gzip_compressor::compress(const char *data, size_t data_length,
-                Callback callback) override {
+                                      bool last, Callback callback) {
   assert(is_valid_);
 
   do {
@@ -2731,8 +2854,7 @@ public:
 
     strm_.avail_in = static_cast<decltype(strm_.avail_in)>(
         std::min(data_length, max_avail_in));
-      strm_.next_in =
+    strm_.next_in = const_cast<Bytef *>(reinterpret_cast<const Bytef *>(data));
-          const_cast<Bytef *>(reinterpret_cast<const Bytef *>(data));
 
     data_length -= strm_.avail_in;
     data += strm_.avail_in;
@@ -2762,14 +2884,7 @@ public:
   return true;
 }
 
-private:
+inline gzip_decompressor::gzip_decompressor() {
-  bool is_valid_ = false;
-  z_stream strm_;
-};
-
-class gzip_decompressor : public decompressor {
-public:
-  gzip_decompressor() {
   std::memset(&strm_, 0, sizeof(strm_));
   strm_.zalloc = Z_NULL;
   strm_.zfree = Z_NULL;
@@ -2782,12 +2897,12 @@ public:
   is_valid_ = inflateInit2(&strm_, 32 + 15) == Z_OK;
 }
 
-  ~gzip_decompressor() { inflateEnd(&strm_); }
+inline gzip_decompressor::~gzip_decompressor() { inflateEnd(&strm_); }
 
-  bool is_valid() const override { return is_valid_; }
+inline bool gzip_decompressor::is_valid() const { return is_valid_; }
 
-  bool decompress(const char *data, size_t data_length,
+inline bool gzip_decompressor::decompress(const char *data, size_t data_length,
-                  Callback callback) override {
+                                          Callback callback) {
   assert(is_valid_);
 
   int ret = Z_OK;
@@ -2798,8 +2913,7 @@ public:
 
     strm_.avail_in = static_cast<decltype(strm_.avail_in)>(
         std::min(data_length, max_avail_in));
-      strm_.next_in =
+    strm_.next_in = const_cast<Bytef *>(reinterpret_cast<const Bytef *>(data));
-          const_cast<Bytef *>(reinterpret_cast<const Bytef *>(data));
 
     data_length -= strm_.avail_in;
     data += strm_.avail_in;
@@ -2828,24 +2942,19 @@ public:
 
   return true;
 }
-
-private:
-  bool is_valid_ = false;
-  z_stream strm_;
-};
 #endif
 
 #ifdef CPPHTTPLIB_BROTLI_SUPPORT
-class brotli_compressor : public compressor {
+inline brotli_compressor::brotli_compressor() {
-public:
-  brotli_compressor() {
   state_ = BrotliEncoderCreateInstance(nullptr, nullptr, nullptr);
 }
 
-  ~brotli_compressor() { BrotliEncoderDestroyInstance(state_); }
+inline brotli_compressor::~brotli_compressor() {
+  BrotliEncoderDestroyInstance(state_);
+}
 
-  bool compress(const char *data, size_t data_length, bool last,
+inline bool brotli_compressor::compress(const char *data, size_t data_length,
-                Callback callback) override {
+                                        bool last, Callback callback) {
   std::array<uint8_t, CPPHTTPLIB_COMPRESSION_BUFSIZ> buff{};
 
   auto operation = last ? BROTLI_OPERATION_FINISH : BROTLI_OPERATION_PROCESS;
@@ -2862,9 +2971,8 @@ public:
     auto available_out = buff.size();
     auto next_out = buff.data();
 
-      if (!BrotliEncoderCompressStream(state_, operation, &available_in,
+    if (!BrotliEncoderCompressStream(state_, operation, &available_in, &next_in,
-                                       &next_in, &available_out, &next_out,
+                                     &available_out, &next_out, nullptr)) {
-                                       nullptr)) {
       return false;
     }
 
@@ -2877,26 +2985,21 @@ public:
   return true;
 }
 
-private:
+inline brotli_decompressor::brotli_decompressor() {
-  BrotliEncoderState *state_ = nullptr;
-};
-
-class brotli_decompressor : public decompressor {
-public:
-  brotli_decompressor() {
   decoder_s = BrotliDecoderCreateInstance(0, 0, 0);
   decoder_r = decoder_s ? BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT
                         : BROTLI_DECODER_RESULT_ERROR;
 }
 
-  ~brotli_decompressor() {
+inline brotli_decompressor::~brotli_decompressor() {
   if (decoder_s) { BrotliDecoderDestroyInstance(decoder_s); }
 }
 
-  bool is_valid() const override { return decoder_s; }
+inline bool brotli_decompressor::is_valid() const { return decoder_s; }
 
-  bool decompress(const char *data, size_t data_length,
+inline bool brotli_decompressor::decompress(const char *data,
-                  Callback callback) override {
+                                            size_t data_length,
+                                            Callback callback) {
   if (decoder_r == BROTLI_DECODER_RESULT_SUCCESS ||
       decoder_r == BROTLI_DECODER_RESULT_ERROR) {
     return 0;
@@ -2925,11 +3028,6 @@ public:
   return decoder_r == BROTLI_DECODER_RESULT_SUCCESS ||
          decoder_r == BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT;
 }
-
-private:
-  BrotliDecoderResult decoder_r;
-  BrotliDecoderState *decoder_s = nullptr;
-};
 #endif
 
 inline bool has_header(const Headers &headers, const char *key) {
@@ -2937,7 +3035,7 @@ inline bool has_header(const Headers &headers, const char *key) {
 }
 
 inline const char *get_header_value(const Headers &headers, const char *key,
-                                    size_t id = 0, const char *def = nullptr) {
+                                    size_t id, const char *def) {
   auto rng = headers.equal_range(key);
   auto it = rng.first;
   std::advance(it, static_cast<ssize_t>(id));
@@ -4060,8 +4158,7 @@ inline std::pair<std::string, std::string> make_range_header(Ranges ranges) {
 
 inline std::pair<std::string, std::string>
 make_basic_authentication_header(const std::string &username,
-                                 const std::string &password,
+                                 const std::string &password, bool is_proxy) {
-                                 bool is_proxy = false) {
   auto field = "Basic " + detail::base64_encode(username + ":" + password);
   auto key = is_proxy ? "Proxy-Authorization" : "Authorization";
   return std::make_pair(key, std::move(field));

test/Makefile

@@ -1,5 +1,5 @@
 #CXX = clang++
-CXXFLAGS = -g -std=c++11 -I.. -I. -Wall -Wextra -Wtype-limits -Wconversion #-fsanitize=address
+CXXFLAGS = -g -std=c++11 -I. -Wall -Wextra -Wtype-limits -Wconversion #-fsanitize=address
 
 OPENSSL_DIR = /usr/local/opt/openssl@1.1
 OPENSSL_SUPPORT = -DCPPHTTPLIB_OPENSSL_SUPPORT -I$(OPENSSL_DIR)/include -L$(OPENSSL_DIR)/lib -lssl -lcrypto
@@ -9,17 +9,27 @@ ZLIB_SUPPORT = -DCPPHTTPLIB_ZLIB_SUPPORT -lz
 BROTLI_DIR = /usr/local/opt/brotli
 BROTLI_SUPPORT = -DCPPHTTPLIB_BROTLI_SUPPORT -I$(BROTLI_DIR)/include -L$(BROTLI_DIR)/lib -lbrotlicommon -lbrotlienc -lbrotlidec
 
-all : test
+TEST_ARGS = gtest/gtest-all.cc gtest/gtest_main.cc $(OPENSSL_SUPPORT) $(ZLIB_SUPPORT) $(BROTLI_SUPPORT) -pthread
+
+all : test test_split
 	./test
+# Note: The intention of test_split is to verify that it works to compile and
+# link the split httplib.h, so there is normally no need to execute it.
 
 proxy : test_proxy
 	./test_proxy
 
-test : test.cc ../httplib.h Makefile cert.pem
+test : test.cc include_httplib.cc ../httplib.h Makefile cert.pem
-	$(CXX) -o test $(CXXFLAGS) test.cc gtest/gtest-all.cc gtest/gtest_main.cc $(OPENSSL_SUPPORT) $(ZLIB_SUPPORT) $(BROTLI_SUPPORT) -pthread
+	$(CXX) -o $@ -I.. $(CXXFLAGS) test.cc include_httplib.cc $(TEST_ARGS)
+
+test_split : test.cc ../httplib.h httplib.cc Makefile cert.pem
+	$(CXX) -o $@ $(CXXFLAGS) test.cc httplib.cc $(TEST_ARGS)
 
 test_proxy : test_proxy.cc ../httplib.h Makefile cert.pem
-	$(CXX) -o test_proxy $(CXXFLAGS) test_proxy.cc gtest/gtest-all.cc gtest/gtest_main.cc $(OPENSSL_SUPPORT) $(ZLIB_SUPPORT) $(BROTLI_SUPPORT) -pthread
+	$(CXX) -o $@ -I.. $(CXXFLAGS) test_proxy.cc $(TEST_ARGS)
+
+httplib.cc : ../httplib.h
+	python3 ../split.py -o .
 
 cert.pem:
 	openssl genrsa 2048 > key.pem
@@ -32,4 +42,4 @@ cert.pem:
 	#c_rehash .
 
 clean:
-	rm -f test test_proxy pem *.0 *.1 *.srl
+	rm -f test test_proxy pem *.0 *.1 *.srl httplib.h httplib.cc

test/include_httplib.cc

@@ -0,0 +1,5 @@
+// The sole purpose of this file is to include httplib.h in a separate
+// compilation unit, thus verifying that inline keywords have not been forgotten
+// when linked together with test.cc.
+
+#include <httplib.h>