From cc0037682bc4d94391ea51eb629a5b4087377123 Mon Sep 17 00:00:00 2001 From: Markus Sattler Date: Wed, 30 Dec 2015 19:38:06 +0100 Subject: [PATCH 1/8] add CHUNKED encoding support too http client (#1324) HTTP Client - fix examples increase default timeout to 5000ms --- .../examples/Authorization/Authorization.ino | 2 +- .../BasicHttpClient/BasicHttpClient.ino | 2 +- .../ReuseConnection/ReuseConnection.ino | 2 +- .../StreamHttpClient/StreamHttpClient.ino | 2 +- .../src/ESP8266HTTPClient.cpp | 178 +++++++++++++----- .../ESP8266HTTPClient/src/ESP8266HTTPClient.h | 15 +- 6 files changed, 146 insertions(+), 55 deletions(-) diff --git a/libraries/ESP8266HTTPClient/examples/Authorization/Authorization.ino b/libraries/ESP8266HTTPClient/examples/Authorization/Authorization.ino index 79a4c3fdf..64955e56b 100644 --- a/libraries/ESP8266HTTPClient/examples/Authorization/Authorization.ino +++ b/libraries/ESP8266HTTPClient/examples/Authorization/Authorization.ino @@ -63,7 +63,7 @@ void loop() { int httpCode = http.GET(); // httpCode will be negative on error - if(httpCode) { + if(httpCode > 0) { // HTTP header has been send and Server response header has been handled USE_SERIAL.printf("[HTTP] GET... code: %d\n", httpCode); diff --git a/libraries/ESP8266HTTPClient/examples/BasicHttpClient/BasicHttpClient.ino b/libraries/ESP8266HTTPClient/examples/BasicHttpClient/BasicHttpClient.ino index d3a5fbc7d..577926716 100644 --- a/libraries/ESP8266HTTPClient/examples/BasicHttpClient/BasicHttpClient.ino +++ b/libraries/ESP8266HTTPClient/examples/BasicHttpClient/BasicHttpClient.ino @@ -51,7 +51,7 @@ void loop() { int httpCode = http.GET(); // httpCode will be negative on error - if(httpCode) { + if(httpCode > 0) { // HTTP header has been send and Server response header has been handled USE_SERIAL.printf("[HTTP] GET... code: %d\n", httpCode); diff --git a/libraries/ESP8266HTTPClient/examples/ReuseConnection/ReuseConnection.ino b/libraries/ESP8266HTTPClient/examples/ReuseConnection/ReuseConnection.ino index 631b9494d..c390515bf 100644 --- a/libraries/ESP8266HTTPClient/examples/ReuseConnection/ReuseConnection.ino +++ b/libraries/ESP8266HTTPClient/examples/ReuseConnection/ReuseConnection.ino @@ -48,7 +48,7 @@ void loop() { //http.begin("192.168.1.12", 80, "/test.html"); int httpCode = http.GET(); - if(httpCode) { + if(httpCode > 0) { USE_SERIAL.printf("[HTTP] GET... code: %d\n", httpCode); // file found at server diff --git a/libraries/ESP8266HTTPClient/examples/StreamHttpClient/StreamHttpClient.ino b/libraries/ESP8266HTTPClient/examples/StreamHttpClient/StreamHttpClient.ino index 2e7f4487d..7fa118254 100644 --- a/libraries/ESP8266HTTPClient/examples/StreamHttpClient/StreamHttpClient.ino +++ b/libraries/ESP8266HTTPClient/examples/StreamHttpClient/StreamHttpClient.ino @@ -50,7 +50,7 @@ void loop() { USE_SERIAL.print("[HTTP] GET...\n"); // start connection and send HTTP header int httpCode = http.GET(); - if(httpCode) { + if(httpCode > 0) { // HTTP header has been send and Server response header has been handled USE_SERIAL.printf("[HTTP] GET... code: %d\n", httpCode); diff --git a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp index 36545fcbb..2a40793d6 100644 --- a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp +++ b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp @@ -40,6 +40,8 @@ HTTPClient::HTTPClient() { _port = 0; _reuse = false; + _tcpTimeout = HTTPCLIENT_DEFAULT_TCP_TIMEOUT; + _https = false; _userAgent = "ESP8266HTTPClient"; @@ -50,7 +52,7 @@ HTTPClient::HTTPClient() { _returnCode = 0; _size = -1; _canReuse = false; - _tcpTimeout = HTTPCLIENT_DEFAULT_TCP_TIMEOUT; + _transferEncoding = HTTPC_TE_IDENTITY; } @@ -468,61 +470,36 @@ int HTTPClient::writeToStream(Stream * stream) { // get length of document (is -1 when Server sends no Content-Length header) int len = _size; - int bytesWritten = 0; + int ret = 0; - size_t buff_size = HTTP_TCP_BUFFER_SIZE; + if(_transferEncoding == HTTPC_TE_IDENTITY) { + ret = writeToStreamDataBlock(stream, len); + } else if(_transferEncoding == HTTPC_TE_CHUNKED) { + while(1) { + String chunkHeader = _tcp->readStringUntil('\n'); + chunkHeader.trim(); // remove \r - // if possible create smaller buffer then HTTP_TCP_BUFFER_SIZE - if((len > 0) && (len < HTTP_TCP_BUFFER_SIZE)) { - buff_size = len; - } - - // create buffer for read - uint8_t * buff = (uint8_t *) malloc(buff_size); - - if(buff) { - // read all data from server - while(connected() && (len > 0 || len == -1)) { - - // get available data size - size_t size = _tcp->available(); - - if(size) { - int c = _tcp->readBytes(buff, ((size > buff_size) ? buff_size : size)); - - // write it to Stream - int w = stream->write(buff, c); - bytesWritten += w; - if(w != c) { - DEBUG_HTTPCLIENT("[HTTP-Client][writeToStream] short write asked for %d but got %d\n", c, w); - break; + // read size of chunk + len = (uint32_t) strtol((const char *) chunkHeader.c_str(), NULL, 16); + DEBUG_HTTPCLIENT("[HTTP-Client] read chunk len: %d\n", len); + if(len > 0) { + int r = writeToStreamDataBlock(stream, len); + if(r < 0) { + // error in writeToStreamDataBlock + return r; } - - if(len > 0) { - len -= c; - } - - delay(0); + ret += r; } else { - delay(1); + break; } + delay(0); } - - free(buff); - - DEBUG_HTTPCLIENT("[HTTP-Client][writeToStream] connection closed or file end (written: %d).\n", bytesWritten); - - if(_size && _size != bytesWritten) { - DEBUG_HTTPCLIENT("[HTTP-Client][writeToStream] bytesWritten %d and size %d mismatch!.\n", bytesWritten, _size); - } - } else { - DEBUG_HTTPCLIENT("[HTTP-Client][writeToStream] too less ram! need %d\n", HTTP_TCP_BUFFER_SIZE); - return HTTPC_ERROR_TOO_LESS_RAM; + return HTTPC_ERROR_ENCODING; } end(); - return bytesWritten; + return ret; } /** @@ -567,6 +544,8 @@ String HTTPClient::errorToString(int error) { return String("no HTTP server"); case HTTPC_ERROR_TOO_LESS_RAM: return String("too less ram"); + case HTTPC_ERROR_ENCODING: + return String("Transfer-Encoding not supported"); default: return String(); } @@ -706,6 +685,7 @@ bool HTTPClient::sendHeader(const char * type) { String header = String(type) + " " + _url + " HTTP/1.1\r\n" "Host: " + _host + "\r\n" "User-Agent: " + _userAgent + "\r\n" + "Accept-Encoding: identity;q=1 chunked;q=0.1 *;q=0\r\n" "Connection: "; if(_reuse) { @@ -733,9 +713,10 @@ int HTTPClient::handleHeaderResponse() { if(!connected()) { return HTTPC_ERROR_NOT_CONNECTED; } - + String transferEncoding; _returnCode = -1; _size = -1; + _transferEncoding = HTTPC_TE_IDENTITY; while(connected()) { size_t len = _tcp->available(); @@ -759,6 +740,10 @@ int HTTPClient::handleHeaderResponse() { _canReuse = headerValue.equalsIgnoreCase("keep-alive"); } + if(headerName.equalsIgnoreCase("Transfer-Encoding")) { + transferEncoding = headerValue; + } + for(size_t i = 0; i < _headerKeysCount; i++) { if(_currentHeaders[i].key.equalsIgnoreCase(headerName)) { _currentHeaders[i].value = headerValue; @@ -769,9 +754,22 @@ int HTTPClient::handleHeaderResponse() { if(headerLine == "") { DEBUG_HTTPCLIENT("[HTTP-Client][handleHeaderResponse] code: %d\n", _returnCode); - if(_size) { + + if(_size > 0) { DEBUG_HTTPCLIENT("[HTTP-Client][handleHeaderResponse] size: %d\n", _size); } + + if(transferEncoding.length() > 0) { + DEBUG_HTTPCLIENT("[HTTP-Client][handleHeaderResponse] Transfer-Encoding: %s\n", transferEncoding.c_str()); + if(transferEncoding.equalsIgnoreCase("chunked")) { + _transferEncoding = HTTPC_TE_CHUNKED; + } else { + return HTTPC_ERROR_ENCODING; + } + } else { + _transferEncoding = HTTPC_TE_IDENTITY; + } + if(_returnCode) { return _returnCode; } else { @@ -787,3 +785,87 @@ int HTTPClient::handleHeaderResponse() { return HTTPC_ERROR_CONNECTION_LOST; } + + + +/** + * + * @param stream + * @param len + * @return + */ +int HTTPClient::writeToStreamDataBlock(Stream * stream, int size) { + int buff_size = HTTP_TCP_BUFFER_SIZE; + int len = size; + int bytesWritten = 0; + + // if possible create smaller buffer then HTTP_TCP_BUFFER_SIZE + if((len > 0) && (len < HTTP_TCP_BUFFER_SIZE)) { + buff_size = len; + } + + // create buffer for read + uint8_t * buff = (uint8_t *) malloc(buff_size); + + if(buff) { + // read all data from server + while(connected() && (len > 0 || len == -1)) { + + // get available data size + size_t sizeAvailable = _tcp->available(); + + if(sizeAvailable) { + + int readBytes = sizeAvailable; + + // read only the asked bytes + if(readBytes > len) { + readBytes = len; + } + + // not read more the buffer can handle + if(readBytes > buff_size) { + readBytes = buff_size; + } + + // read data + int c = _tcp->readBytes(buff, readBytes); + + // write it to Stream + int w = stream->write(buff, c); + bytesWritten += w; + if(w != c) { + DEBUG_HTTPCLIENT("[HTTP-Client][writeToStreamDataBlock] short write asked for %d but got %d\n", c, w); + break; + } + + if(stream->getWriteError()) { + DEBUG_HTTPCLIENT("[HTTP-Client][writeToStreamDataBlock] stream write error %d\n", stream->getWriteError()); + break; + } + + if(len > 0) { + len -= c; + } + + delay(0); + } else { + delay(1); + } + } + + free(buff); + + DEBUG_HTTPCLIENT("[HTTP-Client][writeToStreamDataBlock] connection closed or file end (written: %d).\n", bytesWritten); + + if((size > 0) && (size != bytesWritten)) { + DEBUG_HTTPCLIENT("[HTTP-Client][writeToStreamDataBlock] bytesWritten %d and size %d mismatch!.\n", bytesWritten, size); + } + + } else { + DEBUG_HTTPCLIENT("[HTTP-Client][writeToStreamDataBlock] too less ram! need %d\n", HTTP_TCP_BUFFER_SIZE); + return HTTPC_ERROR_TOO_LESS_RAM; + } + + return bytesWritten; +} diff --git a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.h b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.h index 43c6b19ac..6b32974e1 100644 --- a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.h +++ b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.h @@ -25,13 +25,13 @@ #ifndef ESP8266HTTPClient_H_ #define ESP8266HTTPClient_H_ -//#define DEBUG_HTTPCLIENT(...) Serial1.printf( __VA_ARGS__ ) +#define DEBUG_HTTPCLIENT(...) Serial1.printf( __VA_ARGS__ ) #ifndef DEBUG_HTTPCLIENT #define DEBUG_HTTPCLIENT(...) #endif -#define HTTPCLIENT_DEFAULT_TCP_TIMEOUT (1000) +#define HTTPCLIENT_DEFAULT_TCP_TIMEOUT (5000) /// HTTP client errors #define HTTPC_ERROR_CONNECTION_REFUSED (-1) @@ -42,6 +42,8 @@ #define HTTPC_ERROR_NO_STREAM (-6) #define HTTPC_ERROR_NO_HTTP_SERVER (-7) #define HTTPC_ERROR_TOO_LESS_RAM (-8) +#define HTTPC_ERROR_ENCODING (-9) + /// size for the stream handling #define HTTP_TCP_BUFFER_SIZE (1460) @@ -108,6 +110,11 @@ typedef enum { HTTP_CODE_NETWORK_AUTHENTICATION_REQUIRED = 511 } t_http_codes; +typedef enum { + HTTPC_TE_IDENTITY, + HTTPC_TE_CHUNKED +} transferEncoding_t; + class HTTPClient { public: HTTPClient(); @@ -188,11 +195,13 @@ class HTTPClient { int _returnCode; int _size; bool _canReuse; + transferEncoding_t _transferEncoding; + bool connect(void); bool sendHeader(const char * type); int handleHeaderResponse(); - + int writeToStreamDataBlock(Stream * stream, int len); }; From 288e879a13f7fee1ac283b31a75052f89ad40074 Mon Sep 17 00:00:00 2001 From: Markus Sattler Date: Wed, 30 Dec 2015 21:21:30 +0100 Subject: [PATCH 2/8] disable debug --- libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.h b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.h index 6b32974e1..6b65ed526 100644 --- a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.h +++ b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.h @@ -25,7 +25,7 @@ #ifndef ESP8266HTTPClient_H_ #define ESP8266HTTPClient_H_ -#define DEBUG_HTTPCLIENT(...) Serial1.printf( __VA_ARGS__ ) +//#define DEBUG_HTTPCLIENT(...) Serial1.printf( __VA_ARGS__ ) #ifndef DEBUG_HTTPCLIENT #define DEBUG_HTTPCLIENT(...) From bd7d915bcd812a408559263d203bf0e5ceca1f05 Mon Sep 17 00:00:00 2001 From: Markus Sattler Date: Thu, 31 Dec 2015 12:00:26 +0100 Subject: [PATCH 3/8] rework error handling and detection add retry for short write remove compiler warnings --- .../src/ESP8266HTTPClient.cpp | 111 +++++++++++++++--- .../ESP8266HTTPClient/src/ESP8266HTTPClient.h | 7 +- 2 files changed, 97 insertions(+), 21 deletions(-) diff --git a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp index 2a40793d6..715a4b4d1 100644 --- a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp +++ b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp @@ -298,7 +298,7 @@ int HTTPClient::POST(String payload) { int HTTPClient::sendRequest(const char * type, uint8_t * payload, size_t size) { // connect to server if(!connect()) { - return HTTPC_ERROR_CONNECTION_REFUSED; + return returnError(HTTPC_ERROR_CONNECTION_REFUSED); } if(payload && size > 0) { @@ -307,18 +307,18 @@ int HTTPClient::sendRequest(const char * type, uint8_t * payload, size_t size) { // send Header if(!sendHeader(type)) { - return HTTPC_ERROR_SEND_HEADER_FAILED; + return returnError(HTTPC_ERROR_SEND_HEADER_FAILED); } // send Payload if needed if(payload && size > 0) { if(_tcp->write(&payload[0], size) != size) { - return HTTPC_ERROR_SEND_PAYLOAD_FAILED; + return returnError(HTTPC_ERROR_SEND_PAYLOAD_FAILED); } } // handle Server Response (Header) - return handleHeaderResponse(); + return returnError(handleHeaderResponse()); } /** @@ -348,7 +348,7 @@ int HTTPClient::sendRequest(const char * type, Stream * stream, size_t size) { return HTTPC_ERROR_SEND_HEADER_FAILED; } - size_t buff_size = HTTP_TCP_BUFFER_SIZE; + int buff_size = HTTP_TCP_BUFFER_SIZE; int len = size; int bytesWritten = 0; @@ -371,7 +371,7 @@ int HTTPClient::sendRequest(const char * type, Stream * stream, size_t size) { while(connected() && (stream->available() > -1) && (len > 0 || len == -1)) { // get available data size - size_t s = stream->available(); + int s = stream->available(); if(len) { s = ((s > len) ? len : s); @@ -461,11 +461,11 @@ WiFiClient * HTTPClient::getStreamPtr(void) { int HTTPClient::writeToStream(Stream * stream) { if(!stream) { - return HTTPC_ERROR_NO_STREAM; + return returnError(HTTPC_ERROR_NO_STREAM); } if(!connected()) { - return HTTPC_ERROR_NOT_CONNECTED; + return returnError(HTTPC_ERROR_NOT_CONNECTED); } // get length of document (is -1 when Server sends no Content-Length header) @@ -474,28 +474,56 @@ int HTTPClient::writeToStream(Stream * stream) { if(_transferEncoding == HTTPC_TE_IDENTITY) { ret = writeToStreamDataBlock(stream, len); + + // have we an error? + if(ret < 0) { + return returnError(ret); + } } else if(_transferEncoding == HTTPC_TE_CHUNKED) { + int size = 0; while(1) { + if(!connected()) { + return returnError(HTTPC_ERROR_CONNECTION_LOST); + } String chunkHeader = _tcp->readStringUntil('\n'); + + if(chunkHeader.length() <= 0) { + return returnError(HTTPC_ERROR_READ_TIMEOUT); + } + chunkHeader.trim(); // remove \r // read size of chunk len = (uint32_t) strtol((const char *) chunkHeader.c_str(), NULL, 16); + size += len; DEBUG_HTTPCLIENT("[HTTP-Client] read chunk len: %d\n", len); + + // data left? if(len > 0) { int r = writeToStreamDataBlock(stream, len); if(r < 0) { // error in writeToStreamDataBlock - return r; + return returnError(r); } ret += r; } else { + + // if no length Header use global chunk size + if(_size <= 0) { + _size = size; + } + + // check if we have write all data out + if(ret != _size) { + return returnError(HTTPC_ERROR_STREAM_WRITE); + } break; } + delay(0); } } else { - return HTTPC_ERROR_ENCODING; + return returnError(HTTPC_ERROR_ENCODING); } end(); @@ -546,6 +574,10 @@ String HTTPClient::errorToString(int error) { return String("too less ram"); case HTTPC_ERROR_ENCODING: return String("Transfer-Encoding not supported"); + case HTTPC_ERROR_STREAM_WRITE: + return String("Stream write error"); + case HTTPC_ERROR_READ_TIMEOUT: + return String("read Timeout"); default: return String(); } @@ -829,23 +861,50 @@ int HTTPClient::writeToStreamDataBlock(Stream * stream, int size) { } // read data - int c = _tcp->readBytes(buff, readBytes); + int bytesRead = _tcp->readBytes(buff, readBytes); // write it to Stream - int w = stream->write(buff, c); - bytesWritten += w; - if(w != c) { - DEBUG_HTTPCLIENT("[HTTP-Client][writeToStreamDataBlock] short write asked for %d but got %d\n", c, w); - break; + int bytesWrite = stream->write(buff, bytesRead); + bytesWritten += bytesWrite; + + // are all Bytes a writen to stream ? + if(bytesWrite != bytesRead) { + DEBUG_HTTPCLIENT("[HTTP-Client][writeToStream] short write asked for %d but got %d retry...\n", bytesRead, bytesWrite); + + // check for write error + if(stream->getWriteError()) { + DEBUG_HTTPCLIENT("[HTTP-Client][writeToStreamDataBlock] stream write error %d\n", stream->getWriteError()); + + //reset write error for retry + stream->clearWriteError(); + } + + // some time for the stream + delay(1); + + int leftBytes = (readBytes - bytesWrite); + + // retry to send the missed bytes + bytesWrite = stream->write((buff + bytesWrite), leftBytes); + bytesWritten += bytesWrite; + + if(bytesWrite != leftBytes) { + // failed again + DEBUG_HTTPCLIENT("[HTTP-Client][writeToStream] short write asked for %d but got %d failed.\n", leftBytes, bytesWrite); + free(buff); + return HTTPC_ERROR_STREAM_WRITE; + } } + // check for write error if(stream->getWriteError()) { DEBUG_HTTPCLIENT("[HTTP-Client][writeToStreamDataBlock] stream write error %d\n", stream->getWriteError()); - break; + return HTTPC_ERROR_STREAM_WRITE; } + // count bytes to read left if(len > 0) { - len -= c; + len -= readBytes; } delay(0); @@ -869,3 +928,19 @@ int HTTPClient::writeToStreamDataBlock(Stream * stream, int size) { return bytesWritten; } + +/** + * called to handle error return, may disconnect the connection if still exists + * @param error + * @return error + */ +int HTTPClient::returnError(int error) { + if(error < 0) { + DEBUG_HTTPCLIENT("[HTTP-Client][returnError] error(%d): %s\n", error, errorToString(error).c_str()); + if(connected()) { + DEBUG_HTTPCLIENT("[HTTP-Client][returnError] tcp stop\n"); + _tcp->stop(); + } + } + return error; +} diff --git a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.h b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.h index 6b65ed526..024a27064 100644 --- a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.h +++ b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.h @@ -25,7 +25,7 @@ #ifndef ESP8266HTTPClient_H_ #define ESP8266HTTPClient_H_ -//#define DEBUG_HTTPCLIENT(...) Serial1.printf( __VA_ARGS__ ) +#define DEBUG_HTTPCLIENT(...) Serial1.printf( __VA_ARGS__ ) #ifndef DEBUG_HTTPCLIENT #define DEBUG_HTTPCLIENT(...) @@ -43,7 +43,8 @@ #define HTTPC_ERROR_NO_HTTP_SERVER (-7) #define HTTPC_ERROR_TOO_LESS_RAM (-8) #define HTTPC_ERROR_ENCODING (-9) - +#define HTTPC_ERROR_STREAM_WRITE (-10) +#define HTTPC_ERROR_READ_TIMEOUT (-11) /// size for the stream handling #define HTTP_TCP_BUFFER_SIZE (1460) @@ -197,7 +198,7 @@ class HTTPClient { bool _canReuse; transferEncoding_t _transferEncoding; - + int returnError(int error); bool connect(void); bool sendHeader(const char * type); int handleHeaderResponse(); From 53287f457310162abc6777d8172d661e46becdb2 Mon Sep 17 00:00:00 2001 From: Markus Sattler Date: Thu, 31 Dec 2015 12:14:46 +0100 Subject: [PATCH 4/8] rework sendRequest stream too --- .../src/ESP8266HTTPClient.cpp | 84 ++++++++++++++----- 1 file changed, 64 insertions(+), 20 deletions(-) diff --git a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp index 715a4b4d1..d95242238 100644 --- a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp +++ b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp @@ -331,12 +331,12 @@ int HTTPClient::sendRequest(const char * type, uint8_t * payload, size_t size) { int HTTPClient::sendRequest(const char * type, Stream * stream, size_t size) { if(!stream) { - return HTTPC_ERROR_NO_STREAM; + return returnError(HTTPC_ERROR_NO_STREAM); } // connect to server if(!connect()) { - return HTTPC_ERROR_CONNECTION_REFUSED; + return returnError(HTTPC_ERROR_CONNECTION_REFUSED); } if(size > 0) { @@ -345,7 +345,7 @@ int HTTPClient::sendRequest(const char * type, Stream * stream, size_t size) { // send Header if(!sendHeader(type)) { - return HTTPC_ERROR_SEND_HEADER_FAILED; + return returnError(HTTPC_ERROR_SEND_HEADER_FAILED); } int buff_size = HTTP_TCP_BUFFER_SIZE; @@ -371,25 +371,68 @@ int HTTPClient::sendRequest(const char * type, Stream * stream, size_t size) { while(connected() && (stream->available() > -1) && (len > 0 || len == -1)) { // get available data size - int s = stream->available(); + int sizeAvailable = stream->available(); - if(len) { - s = ((s > len) ? len : s); - } + if(sizeAvailable) { - if(s) { - int c = stream->readBytes(buff, ((s > buff_size) ? buff_size : s)); + int readBytes = sizeAvailable; - // write it to Stream - int w = _tcp->write((const uint8_t *) buff, c); - bytesWritten += w; - if(w != c) { - DEBUG_HTTPCLIENT("[HTTP-Client][sendRequest] short write asked for %d but got %d\n", c, w); - break; + // read only the asked bytes + if(len > 0 && readBytes > len) { + readBytes = len; } + // not read more the buffer can handle + if(readBytes > buff_size) { + readBytes = buff_size; + } + + // read data + int bytesRead = stream->readBytes(buff, readBytes); + + // write it to Stream + int bytesWrite = _tcp->write((const uint8_t *) buff, bytesRead); + bytesWritten += bytesWrite; + + // are all Bytes a writen to stream ? + if(bytesWrite != bytesRead) { + DEBUG_HTTPCLIENT("[HTTP-Client][sendRequest] short write, asked for %d but got %d retry...\n", bytesRead, bytesWrite); + + // check for write error + if(_tcp->getWriteError()) { + DEBUG_HTTPCLIENT("[HTTP-Client][sendRequest] stream write error %d\n", _tcp->getWriteError()); + + //reset write error for retry + _tcp->clearWriteError(); + } + + // some time for the stream + delay(1); + + int leftBytes = (readBytes - bytesWrite); + + // retry to send the missed bytes + bytesWrite = _tcp->write((const uint8_t *) (buff + bytesWrite), leftBytes); + bytesWritten += bytesWrite; + + if(bytesWrite != leftBytes) { + // failed again + DEBUG_HTTPCLIENT("[HTTP-Client][sendRequest] short write, asked for %d but got %d failed.\n", leftBytes, bytesWrite); + free(buff); + return returnError(HTTPC_ERROR_SEND_PAYLOAD_FAILED); + } + } + + // check for write error + if(_tcp->getWriteError()) { + DEBUG_HTTPCLIENT("[HTTP-Client][sendRequest] stream write error %d\n", _tcp->getWriteError()); + free(buff); + return returnError(HTTPC_ERROR_SEND_PAYLOAD_FAILED); + } + + // count bytes to read left if(len > 0) { - len -= c; + len -= readBytes; } delay(0); @@ -403,18 +446,18 @@ int HTTPClient::sendRequest(const char * type, Stream * stream, size_t size) { if(size && (int) size != bytesWritten) { DEBUG_HTTPCLIENT("[HTTP-Client][sendRequest] Stream payload bytesWritten %d and size %d mismatch!.\n", bytesWritten, size); DEBUG_HTTPCLIENT("[HTTP-Client][sendRequest] ERROR SEND PAYLOAD FAILED!"); - return HTTPC_ERROR_SEND_PAYLOAD_FAILED; + return returnError(HTTPC_ERROR_SEND_PAYLOAD_FAILED); } else { DEBUG_HTTPCLIENT("[HTTP-Client][sendRequest] Stream payload written: %d\n", bytesWritten); } } else { DEBUG_HTTPCLIENT("[HTTP-Client][sendRequest] too less ram! need %d\n", HTTP_TCP_BUFFER_SIZE); - return HTTPC_ERROR_TOO_LESS_RAM; + return returnError(HTTPC_ERROR_TOO_LESS_RAM); } // handle Server Response (Header) - return handleHeaderResponse(); + return returnError(handleHeaderResponse()); } /** @@ -851,7 +894,7 @@ int HTTPClient::writeToStreamDataBlock(Stream * stream, int size) { int readBytes = sizeAvailable; // read only the asked bytes - if(readBytes > len) { + if(len > 0 && readBytes > len) { readBytes = len; } @@ -899,6 +942,7 @@ int HTTPClient::writeToStreamDataBlock(Stream * stream, int size) { // check for write error if(stream->getWriteError()) { DEBUG_HTTPCLIENT("[HTTP-Client][writeToStreamDataBlock] stream write error %d\n", stream->getWriteError()); + free(buff); return HTTPC_ERROR_STREAM_WRITE; } From b828f34348d9b1d9af7db1d1997cc7dc2a53a3ed Mon Sep 17 00:00:00 2001 From: Markus Sattler Date: Thu, 31 Dec 2015 12:16:22 +0100 Subject: [PATCH 5/8] some docu --- libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp index d95242238..79923893a 100644 --- a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp +++ b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp @@ -864,10 +864,10 @@ int HTTPClient::handleHeaderResponse() { /** - * - * @param stream - * @param len - * @return + * write one Data Block to Stream + * @param stream Stream * + * @param size int + * @return < 0 = error >= 0 = size written */ int HTTPClient::writeToStreamDataBlock(Stream * stream, int size) { int buff_size = HTTP_TCP_BUFFER_SIZE; From ef748e369a57cde938ab29689843a14a4087227b Mon Sep 17 00:00:00 2001 From: Markus Sattler Date: Thu, 31 Dec 2015 14:02:00 +0100 Subject: [PATCH 6/8] allow downgrade to HTTP 1.0 use HTTP/1.0 for update since the update handler not support any transfer Encoding --- .../src/ESP8266HTTPClient.cpp | 26 +++++++++++++++++-- .../ESP8266HTTPClient/src/ESP8266HTTPClient.h | 3 +++ .../src/ESP8266httpUpdate.cpp | 3 +++ 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp index 79923893a..b1558f3b6 100644 --- a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp +++ b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp @@ -41,6 +41,7 @@ HTTPClient::HTTPClient() { _reuse = false; _tcpTimeout = HTTPCLIENT_DEFAULT_TCP_TIMEOUT; + _useHTTP10 = false; _https = false; @@ -266,6 +267,16 @@ void HTTPClient::setTimeout(uint16_t timeout) { } } + + +/** + * use HTTP1.0 + * @param timeout + */ +void HTTPClient::useHTTP10(bool useHTTP10) { + _useHTTP10 = useHTTP10; +} + /** * send a GET request * @return http code @@ -757,10 +768,17 @@ bool HTTPClient::sendHeader(const char * type) { return false; } - String header = String(type) + " " + _url + " HTTP/1.1\r\n" + String header = String(type) + " " + _url + " HTTP/1."; + + if(_useHTTP10) { + header += "0"; + } else { + header += "1"; + } + + header += "\r\n" "Host: " + _host + "\r\n" "User-Agent: " + _userAgent + "\r\n" - "Accept-Encoding: identity;q=1 chunked;q=0.1 *;q=0\r\n" "Connection: "; if(_reuse) { @@ -770,6 +788,10 @@ bool HTTPClient::sendHeader(const char * type) { } header += "\r\n"; + if(!_useHTTP10) { + header += "Accept-Encoding: identity;q=1 chunked;q=0.1 *;q=0\r\n"; + } + if(_base64Authorization.length()) { header += "Authorization: Basic " + _base64Authorization + "\r\n"; } diff --git a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.h b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.h index 024a27064..aff0ff1ed 100644 --- a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.h +++ b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.h @@ -137,6 +137,8 @@ class HTTPClient { void setAuthorization(const char * auth); void setTimeout(uint16_t timeout); + void useHTTP10(bool usehttp10 = true); + /// request handling int GET(); int POST(uint8_t * payload, size_t size); @@ -180,6 +182,7 @@ class HTTPClient { uint16_t _port; bool _reuse; uint16_t _tcpTimeout; + bool _useHTTP10; String _url; bool _https; diff --git a/libraries/ESP8266httpUpdate/src/ESP8266httpUpdate.cpp b/libraries/ESP8266httpUpdate/src/ESP8266httpUpdate.cpp index f45846440..05b42ea8d 100644 --- a/libraries/ESP8266httpUpdate/src/ESP8266httpUpdate.cpp +++ b/libraries/ESP8266httpUpdate/src/ESP8266httpUpdate.cpp @@ -146,6 +146,9 @@ t_httpUpdate_return ESP8266HTTPUpdate::handleUpdate(HTTPClient * http, const cha t_httpUpdate_return ret = HTTP_UPDATE_FAILED; + // use HTTP/1.0 for update since the update handler not support any transfer Encoding + http->useHTTP10(true); + http->setTimeout(8000); http->setUserAgent("ESP8266-http-Update"); http->addHeader("x-ESP8266-STA-MAC", WiFi.macAddress()); http->addHeader("x-ESP8266-AP-MAC", WiFi.softAPmacAddress()); From 1ab74c1c76372ac1c9a5801800fd10d98d2c1329 Mon Sep 17 00:00:00 2001 From: Markus Sattler Date: Thu, 31 Dec 2015 15:00:00 +0100 Subject: [PATCH 7/8] add missing return HTTPC_ERROR_STREAM_WRITE --- libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp index b1558f3b6..705a5925b 100644 --- a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp +++ b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp @@ -985,6 +985,7 @@ int HTTPClient::writeToStreamDataBlock(Stream * stream, int size) { if((size > 0) && (size != bytesWritten)) { DEBUG_HTTPCLIENT("[HTTP-Client][writeToStreamDataBlock] bytesWritten %d and size %d mismatch!.\n", bytesWritten, size); + return HTTPC_ERROR_STREAM_WRITE; } } else { From 9b06b889c9399247ccbd185f9b07df0263fd1b0d Mon Sep 17 00:00:00 2001 From: Markus Sattler Date: Thu, 31 Dec 2015 17:10:38 +0100 Subject: [PATCH 8/8] disable HTTP client debug --- libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.h b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.h index aff0ff1ed..bcd1cc3a1 100644 --- a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.h +++ b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.h @@ -25,7 +25,7 @@ #ifndef ESP8266HTTPClient_H_ #define ESP8266HTTPClient_H_ -#define DEBUG_HTTPCLIENT(...) Serial1.printf( __VA_ARGS__ ) +//#define DEBUG_HTTPCLIENT(...) Serial1.printf( __VA_ARGS__ ) #ifndef DEBUG_HTTPCLIENT #define DEBUG_HTTPCLIENT(...)