From cc0037682bc4d94391ea51eb629a5b4087377123 Mon Sep 17 00:00:00 2001 From: Markus Sattler Date: Wed, 30 Dec 2015 19:38:06 +0100 Subject: [PATCH] add CHUNKED encoding support too http client (#1324) HTTP Client - fix examples increase default timeout to 5000ms --- .../examples/Authorization/Authorization.ino | 2 +- .../BasicHttpClient/BasicHttpClient.ino | 2 +- .../ReuseConnection/ReuseConnection.ino | 2 +- .../StreamHttpClient/StreamHttpClient.ino | 2 +- .../src/ESP8266HTTPClient.cpp | 178 +++++++++++++----- .../ESP8266HTTPClient/src/ESP8266HTTPClient.h | 15 +- 6 files changed, 146 insertions(+), 55 deletions(-) diff --git a/libraries/ESP8266HTTPClient/examples/Authorization/Authorization.ino b/libraries/ESP8266HTTPClient/examples/Authorization/Authorization.ino index 79a4c3fdf..64955e56b 100644 --- a/libraries/ESP8266HTTPClient/examples/Authorization/Authorization.ino +++ b/libraries/ESP8266HTTPClient/examples/Authorization/Authorization.ino @@ -63,7 +63,7 @@ void loop() { int httpCode = http.GET(); // httpCode will be negative on error - if(httpCode) { + if(httpCode > 0) { // HTTP header has been send and Server response header has been handled USE_SERIAL.printf("[HTTP] GET... code: %d\n", httpCode); diff --git a/libraries/ESP8266HTTPClient/examples/BasicHttpClient/BasicHttpClient.ino b/libraries/ESP8266HTTPClient/examples/BasicHttpClient/BasicHttpClient.ino index d3a5fbc7d..577926716 100644 --- a/libraries/ESP8266HTTPClient/examples/BasicHttpClient/BasicHttpClient.ino +++ b/libraries/ESP8266HTTPClient/examples/BasicHttpClient/BasicHttpClient.ino @@ -51,7 +51,7 @@ void loop() { int httpCode = http.GET(); // httpCode will be negative on error - if(httpCode) { + if(httpCode > 0) { // HTTP header has been send and Server response header has been handled USE_SERIAL.printf("[HTTP] GET... code: %d\n", httpCode); diff --git a/libraries/ESP8266HTTPClient/examples/ReuseConnection/ReuseConnection.ino b/libraries/ESP8266HTTPClient/examples/ReuseConnection/ReuseConnection.ino index 631b9494d..c390515bf 100644 --- a/libraries/ESP8266HTTPClient/examples/ReuseConnection/ReuseConnection.ino +++ b/libraries/ESP8266HTTPClient/examples/ReuseConnection/ReuseConnection.ino @@ -48,7 +48,7 @@ void loop() { //http.begin("192.168.1.12", 80, "/test.html"); int httpCode = http.GET(); - if(httpCode) { + if(httpCode > 0) { USE_SERIAL.printf("[HTTP] GET... code: %d\n", httpCode); // file found at server diff --git a/libraries/ESP8266HTTPClient/examples/StreamHttpClient/StreamHttpClient.ino b/libraries/ESP8266HTTPClient/examples/StreamHttpClient/StreamHttpClient.ino index 2e7f4487d..7fa118254 100644 --- a/libraries/ESP8266HTTPClient/examples/StreamHttpClient/StreamHttpClient.ino +++ b/libraries/ESP8266HTTPClient/examples/StreamHttpClient/StreamHttpClient.ino @@ -50,7 +50,7 @@ void loop() { USE_SERIAL.print("[HTTP] GET...\n"); // start connection and send HTTP header int httpCode = http.GET(); - if(httpCode) { + if(httpCode > 0) { // HTTP header has been send and Server response header has been handled USE_SERIAL.printf("[HTTP] GET... code: %d\n", httpCode); diff --git a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp index 36545fcbb..2a40793d6 100644 --- a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp +++ b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp @@ -40,6 +40,8 @@ HTTPClient::HTTPClient() { _port = 0; _reuse = false; + _tcpTimeout = HTTPCLIENT_DEFAULT_TCP_TIMEOUT; + _https = false; _userAgent = "ESP8266HTTPClient"; @@ -50,7 +52,7 @@ HTTPClient::HTTPClient() { _returnCode = 0; _size = -1; _canReuse = false; - _tcpTimeout = HTTPCLIENT_DEFAULT_TCP_TIMEOUT; + _transferEncoding = HTTPC_TE_IDENTITY; } @@ -468,61 +470,36 @@ int HTTPClient::writeToStream(Stream * stream) { // get length of document (is -1 when Server sends no Content-Length header) int len = _size; - int bytesWritten = 0; + int ret = 0; - size_t buff_size = HTTP_TCP_BUFFER_SIZE; + if(_transferEncoding == HTTPC_TE_IDENTITY) { + ret = writeToStreamDataBlock(stream, len); + } else if(_transferEncoding == HTTPC_TE_CHUNKED) { + while(1) { + String chunkHeader = _tcp->readStringUntil('\n'); + chunkHeader.trim(); // remove \r - // if possible create smaller buffer then HTTP_TCP_BUFFER_SIZE - if((len > 0) && (len < HTTP_TCP_BUFFER_SIZE)) { - buff_size = len; - } - - // create buffer for read - uint8_t * buff = (uint8_t *) malloc(buff_size); - - if(buff) { - // read all data from server - while(connected() && (len > 0 || len == -1)) { - - // get available data size - size_t size = _tcp->available(); - - if(size) { - int c = _tcp->readBytes(buff, ((size > buff_size) ? buff_size : size)); - - // write it to Stream - int w = stream->write(buff, c); - bytesWritten += w; - if(w != c) { - DEBUG_HTTPCLIENT("[HTTP-Client][writeToStream] short write asked for %d but got %d\n", c, w); - break; + // read size of chunk + len = (uint32_t) strtol((const char *) chunkHeader.c_str(), NULL, 16); + DEBUG_HTTPCLIENT("[HTTP-Client] read chunk len: %d\n", len); + if(len > 0) { + int r = writeToStreamDataBlock(stream, len); + if(r < 0) { + // error in writeToStreamDataBlock + return r; } - - if(len > 0) { - len -= c; - } - - delay(0); + ret += r; } else { - delay(1); + break; } + delay(0); } - - free(buff); - - DEBUG_HTTPCLIENT("[HTTP-Client][writeToStream] connection closed or file end (written: %d).\n", bytesWritten); - - if(_size && _size != bytesWritten) { - DEBUG_HTTPCLIENT("[HTTP-Client][writeToStream] bytesWritten %d and size %d mismatch!.\n", bytesWritten, _size); - } - } else { - DEBUG_HTTPCLIENT("[HTTP-Client][writeToStream] too less ram! need %d\n", HTTP_TCP_BUFFER_SIZE); - return HTTPC_ERROR_TOO_LESS_RAM; + return HTTPC_ERROR_ENCODING; } end(); - return bytesWritten; + return ret; } /** @@ -567,6 +544,8 @@ String HTTPClient::errorToString(int error) { return String("no HTTP server"); case HTTPC_ERROR_TOO_LESS_RAM: return String("too less ram"); + case HTTPC_ERROR_ENCODING: + return String("Transfer-Encoding not supported"); default: return String(); } @@ -706,6 +685,7 @@ bool HTTPClient::sendHeader(const char * type) { String header = String(type) + " " + _url + " HTTP/1.1\r\n" "Host: " + _host + "\r\n" "User-Agent: " + _userAgent + "\r\n" + "Accept-Encoding: identity;q=1 chunked;q=0.1 *;q=0\r\n" "Connection: "; if(_reuse) { @@ -733,9 +713,10 @@ int HTTPClient::handleHeaderResponse() { if(!connected()) { return HTTPC_ERROR_NOT_CONNECTED; } - + String transferEncoding; _returnCode = -1; _size = -1; + _transferEncoding = HTTPC_TE_IDENTITY; while(connected()) { size_t len = _tcp->available(); @@ -759,6 +740,10 @@ int HTTPClient::handleHeaderResponse() { _canReuse = headerValue.equalsIgnoreCase("keep-alive"); } + if(headerName.equalsIgnoreCase("Transfer-Encoding")) { + transferEncoding = headerValue; + } + for(size_t i = 0; i < _headerKeysCount; i++) { if(_currentHeaders[i].key.equalsIgnoreCase(headerName)) { _currentHeaders[i].value = headerValue; @@ -769,9 +754,22 @@ int HTTPClient::handleHeaderResponse() { if(headerLine == "") { DEBUG_HTTPCLIENT("[HTTP-Client][handleHeaderResponse] code: %d\n", _returnCode); - if(_size) { + + if(_size > 0) { DEBUG_HTTPCLIENT("[HTTP-Client][handleHeaderResponse] size: %d\n", _size); } + + if(transferEncoding.length() > 0) { + DEBUG_HTTPCLIENT("[HTTP-Client][handleHeaderResponse] Transfer-Encoding: %s\n", transferEncoding.c_str()); + if(transferEncoding.equalsIgnoreCase("chunked")) { + _transferEncoding = HTTPC_TE_CHUNKED; + } else { + return HTTPC_ERROR_ENCODING; + } + } else { + _transferEncoding = HTTPC_TE_IDENTITY; + } + if(_returnCode) { return _returnCode; } else { @@ -787,3 +785,87 @@ int HTTPClient::handleHeaderResponse() { return HTTPC_ERROR_CONNECTION_LOST; } + + + +/** + * + * @param stream + * @param len + * @return + */ +int HTTPClient::writeToStreamDataBlock(Stream * stream, int size) { + int buff_size = HTTP_TCP_BUFFER_SIZE; + int len = size; + int bytesWritten = 0; + + // if possible create smaller buffer then HTTP_TCP_BUFFER_SIZE + if((len > 0) && (len < HTTP_TCP_BUFFER_SIZE)) { + buff_size = len; + } + + // create buffer for read + uint8_t * buff = (uint8_t *) malloc(buff_size); + + if(buff) { + // read all data from server + while(connected() && (len > 0 || len == -1)) { + + // get available data size + size_t sizeAvailable = _tcp->available(); + + if(sizeAvailable) { + + int readBytes = sizeAvailable; + + // read only the asked bytes + if(readBytes > len) { + readBytes = len; + } + + // not read more the buffer can handle + if(readBytes > buff_size) { + readBytes = buff_size; + } + + // read data + int c = _tcp->readBytes(buff, readBytes); + + // write it to Stream + int w = stream->write(buff, c); + bytesWritten += w; + if(w != c) { + DEBUG_HTTPCLIENT("[HTTP-Client][writeToStreamDataBlock] short write asked for %d but got %d\n", c, w); + break; + } + + if(stream->getWriteError()) { + DEBUG_HTTPCLIENT("[HTTP-Client][writeToStreamDataBlock] stream write error %d\n", stream->getWriteError()); + break; + } + + if(len > 0) { + len -= c; + } + + delay(0); + } else { + delay(1); + } + } + + free(buff); + + DEBUG_HTTPCLIENT("[HTTP-Client][writeToStreamDataBlock] connection closed or file end (written: %d).\n", bytesWritten); + + if((size > 0) && (size != bytesWritten)) { + DEBUG_HTTPCLIENT("[HTTP-Client][writeToStreamDataBlock] bytesWritten %d and size %d mismatch!.\n", bytesWritten, size); + } + + } else { + DEBUG_HTTPCLIENT("[HTTP-Client][writeToStreamDataBlock] too less ram! need %d\n", HTTP_TCP_BUFFER_SIZE); + return HTTPC_ERROR_TOO_LESS_RAM; + } + + return bytesWritten; +} diff --git a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.h b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.h index 43c6b19ac..6b32974e1 100644 --- a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.h +++ b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.h @@ -25,13 +25,13 @@ #ifndef ESP8266HTTPClient_H_ #define ESP8266HTTPClient_H_ -//#define DEBUG_HTTPCLIENT(...) Serial1.printf( __VA_ARGS__ ) +#define DEBUG_HTTPCLIENT(...) Serial1.printf( __VA_ARGS__ ) #ifndef DEBUG_HTTPCLIENT #define DEBUG_HTTPCLIENT(...) #endif -#define HTTPCLIENT_DEFAULT_TCP_TIMEOUT (1000) +#define HTTPCLIENT_DEFAULT_TCP_TIMEOUT (5000) /// HTTP client errors #define HTTPC_ERROR_CONNECTION_REFUSED (-1) @@ -42,6 +42,8 @@ #define HTTPC_ERROR_NO_STREAM (-6) #define HTTPC_ERROR_NO_HTTP_SERVER (-7) #define HTTPC_ERROR_TOO_LESS_RAM (-8) +#define HTTPC_ERROR_ENCODING (-9) + /// size for the stream handling #define HTTP_TCP_BUFFER_SIZE (1460) @@ -108,6 +110,11 @@ typedef enum { HTTP_CODE_NETWORK_AUTHENTICATION_REQUIRED = 511 } t_http_codes; +typedef enum { + HTTPC_TE_IDENTITY, + HTTPC_TE_CHUNKED +} transferEncoding_t; + class HTTPClient { public: HTTPClient(); @@ -188,11 +195,13 @@ class HTTPClient { int _returnCode; int _size; bool _canReuse; + transferEncoding_t _transferEncoding; + bool connect(void); bool sendHeader(const char * type); int handleHeaderResponse(); - + int writeToStreamDataBlock(Stream * stream, int len); };