Add support for chunked response bodies

keywords.txt

@@ -30,6 +30,7 @@ endOfHeadersReached	KEYWORD2
 endOfBodyReached	KEYWORD2
 completed	KEYWORD2
 contentLength	KEYWORD2
+isChunked	KEYWORD2
 connectionKeepAlive	KEYWORD2
 noDefaultRequestHeaders	KEYWORD2
 headerAvailable	KEYWORD2

src/HttpClient.cpp

@@ -8,6 +8,7 @@
 // Initialize constants
 const char* HttpClient::kUserAgent = "Arduino/2.2.0";
 const char* HttpClient::kContentLengthPrefix = HTTP_HEADER_CONTENT_LENGTH ": ";
+const char* HttpClient::kTransferEncodingChunked = HTTP_HEADER_TRANSFER_ENCODING ": " HTTP_HEADER_VALUE_CHUNKED;
 
 HttpClient::HttpClient(Client& aClient, const char* aServerName, uint16_t aServerPort)
  : iClient(&aClient), iServerName(aServerName), iServerAddress(), iServerPort(aServerPort),
@@ -35,6 +36,9 @@ void HttpClient::resetState()
   iContentLength = kNoContentLengthHeader;
   iBodyLengthConsumed = 0;
   iContentLengthPtr = kContentLengthPrefix;
+  iTransferEncodingChunkedPtr = kTransferEncodingChunked;
+  iIsChunked = false;
+  iChunkLength = 0;
   iHttpResponseTimeout = kHttpResponseTimeout;
 }
 
@@ -62,7 +66,7 @@ void HttpClient::beginRequest()
 int HttpClient::startRequest(const char* aURLPath, const char* aHttpMethod, 
                                 const char* aContentType, int aContentLength, const byte aBody[])
 {
-    if (iState == eReadingBody)
+    if (iState == eReadingBody || iState == eReadingChunkLength || iState == eReadingBodyChunk)
     {
         flushClientRx();
 
@@ -528,6 +532,11 @@ int HttpClient::skipResponseHeaders()
     }
 }
 
+bool HttpClient::endOfHeadersReached()
+{
+    return (iState == eReadingBody || iState == eReadingChunkLength || iState == eReadingBodyChunk);
+};
+
 int HttpClient::contentLength()
 {
     // skip the response headers, if they haven't been read already 
@@ -590,8 +599,65 @@ bool HttpClient::endOfBodyReached()
     return false;
 }
 
+int HttpClient::available()
+{
+    if (iState == eReadingChunkLength)
+    {
+        while (iClient->available())
+        {
+            char c = iClient->read();
+
+            if (c == '\n')
+            {
+                iState = eReadingBodyChunk;
+                break;
+            }
+            else if (c == '\r')
+            {
+                // no-op
+            }
+            else if (isHexadecimalDigit(c))
+            {
+                char digit[2] = {c, '\0'};
+
+                iChunkLength = (iChunkLength * 16) + strtol(digit, NULL, 16);
+            }
+        }
+    }
+
+    if (iState == eReadingBodyChunk && iChunkLength == 0)
+    {
+        iState = eReadingChunkLength;
+    }
+    
+    if (iState == eReadingChunkLength)
+    {
+        return 0;
+    }
+    
+    int clientAvailable = iClient->available();
+
+    if (iState == eReadingBodyChunk)
+    {
+        return min(clientAvailable, iChunkLength);
+    }
+    else
+    {
+        return clientAvailable;
+    }
+}
+
+
 int HttpClient::read()
 {
+    if (iState == eReadingBodyChunk)
+    {
+        if (!available())
+        {
+            return -1;
+        }
+    }
+
     int ret = iClient->read();
     if (ret >= 0)
     {
@@ -601,6 +667,16 @@ int HttpClient::read()
             // So keep track of how many bytes are left
             iBodyLengthConsumed++;
         }
+
+        if (iState == eReadingBodyChunk)
+        {
+            iChunkLength--;
+
+            if (iChunkLength == 0)
+            {
+                iState = eReadingChunkLength;
+            }
+        }
     }
     return ret;
 }
@@ -714,7 +790,18 @@ int HttpClient::readHeader()
                 iBodyLengthConsumed = 0;
             }
         }
-        else if ((iContentLengthPtr == kContentLengthPrefix) && (c == '\r'))
+        else if (*iTransferEncodingChunkedPtr == c)
+        {
+            // This character matches, just move along
+            iTransferEncodingChunkedPtr++;
+            if (*iTransferEncodingChunkedPtr == '\0')
+            {
+                // We've reached the end of the Transfer Encoding: chunked header
+                iIsChunked = true;
+                iState = eSkipToEndOfHeader;
+            }
+        }
+        else if (((iContentLengthPtr == kContentLengthPrefix) && (iTransferEncodingChunkedPtr == kTransferEncodingChunked)) && (c == '\r'))
         {
             // We've found a '\r' at the start of a line, so this is probably
             // the end of the headers
@@ -722,7 +809,7 @@ int HttpClient::readHeader()
         }
         else
         {
-            // This isn't the Content-Length header, skip to the end of the line
+            // This isn't the Content-Length or Transfer Encoding chunked header, skip to the end of the line
             iState = eSkipToEndOfHeader;
         }
         break;
@@ -742,7 +829,15 @@ int HttpClient::readHeader()
     case eLineStartingCRFound:
         if (c == '\n')
         {
-            iState = eReadingBody;
+            if (iIsChunked)
+            {
+                iState = eReadingChunkLength;
+                iChunkLength = 0;
+            }
+            else
+            {
+                iState = eReadingBody;
+            }
         }
         break;
     default:
@@ -755,6 +850,7 @@ int HttpClient::readHeader()
         // We've got to the end of this line, start processing again
         iState = eStatusCodeRead;
         iContentLengthPtr = kContentLengthPrefix;
+        iTransferEncodingChunkedPtr = kTransferEncodingChunked;
     }
     // And return the character read to whoever wants it
     return c;

src/HttpClient.h

@@ -34,7 +34,9 @@ static const int HTTP_ERROR_INVALID_RESPONSE =-4;
 #define HTTP_HEADER_CONTENT_LENGTH "Content-Length"
 #define HTTP_HEADER_CONTENT_TYPE   "Content-Type"
 #define HTTP_HEADER_CONNECTION     "Connection"
+#define HTTP_HEADER_TRANSFER_ENCODING "Transfer-Encoding"
 #define HTTP_HEADER_USER_AGENT     "User-Agent"
+#define HTTP_HEADER_VALUE_CHUNKED  "chunked"
 
 class HttpClient : public Client
 {
@@ -247,7 +249,7 @@ public:
     /** Test whether all of the response headers have been consumed.
       @return true if we are now processing the response body, else false
     */
-    bool endOfHeadersReached() { return (iState == eReadingBody); };
+    bool endOfHeadersReached();
 
     /** Test whether the end of the body has been reached.
       Only works if the Content-Length header was returned by the server
@@ -265,6 +267,11 @@ public:
     */
     int contentLength();
 
+    /** Returns if the response body is chunked
+      @return true if response body is chunked, false otherwise
+    */
+    int isChunked() { return iIsChunked; }
+
     /** Return the response body as a String
       Also skips response headers if they have not been read already
       MUST be called after responseStatusCode()
@@ -286,7 +293,7 @@ public:
     virtual size_t write(uint8_t aByte) { if (iState < eRequestSent) { finishHeaders(); }; return iClient-> write(aByte); };
     virtual size_t write(const uint8_t *aBuffer, size_t aSize) { if (iState < eRequestSent) { finishHeaders(); }; return iClient->write(aBuffer, aSize); };
     // Inherited from Stream
-    virtual int available() { return iClient->available(); };
+    virtual int available();
     /** Read the next byte from the server.
       @return Byte read or -1 if there are no bytes available.
     */
@@ -332,6 +339,7 @@ protected:
     // processing)
     static const int kHttpResponseTimeout = 30*1000;
     static const char* kContentLengthPrefix;
+    static const char* kTransferEncodingChunked;
     typedef enum {
         eIdle,
         eRequestStarted,
@@ -341,7 +349,9 @@ protected:
         eReadingContentLength,
         eSkipToEndOfHeader,
         eLineStartingCRFound,
-        eReadingBody
+        eReadingBody,
+        eReadingChunkLength,
+        eReadingBodyChunk
     } tHttpState;
     // Client we're using
     Client* iClient;
@@ -360,6 +370,12 @@ protected:
     int iBodyLengthConsumed;
     // How far through a Content-Length header prefix we are
     const char* iContentLengthPtr;
+    // How far through a Transfer-Encoding chunked header we are
+    const char* iTransferEncodingChunkedPtr;
+    // Stores if the response body is chunked
+    bool iIsChunked;
+    // Stores the value of the current chunk length, if present
+    int iChunkLength;
     uint32_t iHttpResponseTimeout;
     bool iConnectionClose;
     bool iSendDefaultRequestHeaders;