optionally allow redirects on HTTPClient & OTA updates (#5009)

* optionally allow redirects on http OTA updates

* Refactored HTTPClient::begin(url...) & setURL functions, now only beginInternal parses URL, sets ports
Added HTTPRedirect example.

* fix indentation for style check

* add space after while for style check

* don't use deprecated begin method in redirect example

* moved redirect handling code to HTTPClient.
only GET and HEAD requests are currently handled automatically
Redirects that fail to be automatically handled return the redirect code as before

* added support for POST/303 redirect
added device redirect tests

* add missing getLocation() implementation

* if the new location is only a path then only update the URI

libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp

@@ -139,6 +139,7 @@ void HTTPClient::clear()
     _size = -1;
     _headers = "";
     _payload.reset();
+    _location = "";
 }
 
 
@@ -217,7 +218,6 @@ bool HTTPClient::begin(String url, String httpsFingerprint)
         end();
     }
 
-    _port = 443;
     if (httpsFingerprint.length() == 0) {
         return false;
     }
@@ -238,7 +238,6 @@ bool HTTPClient::begin(String url, const uint8_t httpsFingerprint[20])
         end();
     }
 
-    _port = 443;
     if (!beginInternal(url, "https")) {
         return false;
     }
@@ -264,7 +263,6 @@ bool HTTPClient::begin(String url)
         end();
     }
 
-    _port = 80;
     if (!beginInternal(url, "http")) {
         return false;
     }
@@ -288,6 +286,17 @@ bool HTTPClient::beginInternal(String url, const char* expectedProtocol)
     _protocol = url.substring(0, index);
     url.remove(0, (index + 3)); // remove http:// or https://
 
+    if (_protocol == "http") {
+        // set default port for 'http'
+        _port = 80;
+    } else if (_protocol == "https") {
+        // set default port for 'https'
+        _port = 443;
+    } else {
+        DEBUG_HTTPCLIENT("[HTTP-Client][begin] unsupported protocol: %s\n", _protocol.c_str());
+        return false;
+    }
+
     index = url.indexOf('/');
     String host = url.substring(0, index);
     url.remove(0, index); // remove host part
@@ -312,7 +321,7 @@ bool HTTPClient::beginInternal(String url, const char* expectedProtocol)
     }
     _uri = url;
 
-    if (_protocol != expectedProtocol) {
+    if ( expectedProtocol != nullptr && _protocol != expectedProtocol) {
         DEBUG_HTTPCLIENT("[HTTP-Client][begin] unexpected protocol: %s, expected %s\n", _protocol.c_str(), expectedProtocol);
         return false;
     }
@@ -402,13 +411,14 @@ void HTTPClient::end(void)
 {
     disconnect();
     clear();
+    _redirectCount = 0;
 }
 
 /**
  * disconnect
  * close the TCP socket
  */
-void HTTPClient::disconnect()
+void HTTPClient::disconnect(bool preserveClient)
 {
     if(connected()) {
         if(_client->available() > 0) {
@@ -424,7 +434,9 @@ void HTTPClient::disconnect()
             DEBUG_HTTPCLIENT("[HTTP-Client][end] tcp stop\n");
             if(_client) {
                 _client->stop();
-                _client = nullptr;
+                if (!preserveClient) {
+                    _client = nullptr;
+                }
             }
 #if HTTPCLIENT_1_1_COMPATIBLE
             if(_tcpDeprecated) {
@@ -507,6 +519,43 @@ void HTTPClient::setTimeout(uint16_t timeout)
     }
 }
 
+/**
+ * set the URL to a new value. Handy for following redirects.
+ * @param url
+ */
+bool HTTPClient::setURL(String url)
+{
+    // if the new location is only a path then only update the URI
+    if (_location.startsWith("/")) {
+        _uri = _location;
+        clear();
+        return true;
+    }
+
+    if (!url.startsWith(_protocol + ":")) {
+        DEBUG_HTTPCLIENT("[HTTP-Client][setURL] new URL not the same protocol, expected '%s', URL: '%s'\n", _protocol.c_str(), url.c_str());
+        return false;
+    }
+    // disconnect but preserve _client
+    disconnect(true);
+    clear();
+    return beginInternal(url, nullptr);
+}
+
+/**
+ * set true to follow redirects.
+ * @param follow
+ */
+void HTTPClient::setFollowRedirects(bool follow)
+{
+    _followRedirects = follow;
+}
+
+void HTTPClient::setRedirectLimit(uint16_t limit)
+{
+    _redirectLimit = limit;
+}
+
 /**
  * use HTTP1.0
  * @param timeout
@@ -589,29 +638,82 @@ int HTTPClient::sendRequest(const char * type, String payload)
  */
 int HTTPClient::sendRequest(const char * type, uint8_t * payload, size_t size)
 {
-    // connect to server
+    bool redirect = false;
-    if(!connect()) {
+    int code = 0;
-        return returnError(HTTPC_ERROR_CONNECTION_REFUSED);
+    do {
-    }
+        // wipe out any existing headers from previous request
+        for(size_t i = 0; i < _headerKeysCount; i++) {
+            if (_currentHeaders[i].value.length() > 0) {
+                _currentHeaders[i].value = "";
+            }
+        }
 
-    if(payload && size > 0) {
+        redirect = false;
-        addHeader(F("Content-Length"), String(size));
+        DEBUG_HTTPCLIENT("[HTTP-Client][sendRequest] type: '%s' redirCount: %d\n", type, _redirectCount);
-    }
 
-    // send Header
+        // connect to server
-    if(!sendHeader(type)) {
+        if(!connect()) {
-        return returnError(HTTPC_ERROR_SEND_HEADER_FAILED);
+            return returnError(HTTPC_ERROR_CONNECTION_REFUSED);
-    }
+        }
 
-    // send Payload if needed
+        if(payload && size > 0) {
-    if(payload && size > 0) {
+            addHeader(F("Content-Length"), String(size));
-        if(_client->write(&payload[0], size) != size) {
+        }
-            return returnError(HTTPC_ERROR_SEND_PAYLOAD_FAILED);
+
+        // send Header
+        if(!sendHeader(type)) {
+            return returnError(HTTPC_ERROR_SEND_HEADER_FAILED);
+        }
+
+        // send Payload if needed
+        if(payload && size > 0) {
+            if(_client->write(&payload[0], size) != size) {
+                return returnError(HTTPC_ERROR_SEND_PAYLOAD_FAILED);
+            }
+        }
+
+        // handle Server Response (Header)
+        code = handleHeaderResponse();
+
+        //
+        // We can follow redirects for 301/302/307 for GET and HEAD requests and
+        // and we have not exceeded the redirect limit preventing an infinite
+        // redirect loop.
+        //
+        // https://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html
+        //
+        if (_followRedirects &&
+                (_redirectCount < _redirectLimit) &&
+                (_location.length() > 0) &&
+                (code == 301 || code == 302 || code == 307) &&
+                (!strcmp(type, "GET") || !strcmp(type, "HEAD"))
+                ) {
+            _redirectCount += 1; // increment the count for redirect.
+            redirect = true;
+            DEBUG_HTTPCLIENT("[HTTP-Client][sendRequest] following redirect:: '%s' redirCount: %d\n", _location.c_str(), _redirectCount);
+            if (!setURL(_location)) {
+                // return the redirect instead of handling on failure of setURL()
+                redirect = false;
+            }
+        }
+
+    } while (redirect);
+
+    // handle 303 redirect for non GET/HEAD by changing to GET and requesting new url
+    if (_followRedirects &&
+            (_redirectCount < _redirectLimit) &&
+            (_location.length() > 0) &&
+            (code == 303) &&
+            strcmp(type, "GET") && strcmp(type, "HEAD")
+            ) {
+        _redirectCount += 1;
+        if (setURL(_location)) {
+            code = sendRequest("GET");
         }
     }
 
     // handle Server Response (Header)
-    return returnError(handleHeaderResponse());
+    return returnError(code);
 }
 
 /**
@@ -762,6 +864,14 @@ int HTTPClient::getSize(void)
     return _size;
 }
 
+/**
+ * Location if redirect
+ */
+const String& HTTPClient::getLocation(void)
+{
+    return _location;
+}
+
 /**
  * returns the stream of the tcp connection
  * @return WiFiClient
@@ -1173,6 +1283,10 @@ int HTTPClient::handleHeaderResponse()
                     transferEncoding = headerValue;
                 }
 
+                if(headerName.equalsIgnoreCase("Location")) {
+                    _location = headerValue;
+                }
+
                 for(size_t i = 0; i < _headerKeysCount; i++) {
                     if(_currentHeaders[i].key.equalsIgnoreCase(headerName)) {
                         if (_currentHeaders[i].value != "") {

libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.h

@@ -173,7 +173,9 @@ public:
     void setAuthorization(const char * user, const char * password);
     void setAuthorization(const char * auth);
     void setTimeout(uint16_t timeout);
-
+    void setFollowRedirects(bool follow);
+    void setRedirectLimit(uint16_t limit); // max redirects to follow for a single request
+    bool setURL(String url); // handy for handling redirects
     void useHTTP10(bool usehttp10 = true);
 
     /// request handling
@@ -200,12 +202,12 @@ public:
 
 
     int getSize(void);
+    const String& getLocation(void); // Location header from redirect if 3XX
 
     WiFiClient& getStream(void);
     WiFiClient* getStreamPtr(void);
     int writeToStream(Stream* stream);
     const String& getString(void);
-
     static String errorToString(int error);
 
 protected:
@@ -215,7 +217,7 @@ protected:
     };
 
     bool beginInternal(String url, const char* expectedProtocol);
-    void disconnect();
+    void disconnect(bool preserveClient = false);
     void clear();
     int returnError(int error);
     bool connect(void);
@@ -250,6 +252,10 @@ protected:
     int _returnCode = 0;
     int _size = -1;
     bool _canReuse = false;
+    bool _followRedirects = false;
+    uint16_t _redirectCount = 0;
+    uint16_t _redirectLimit = 10;
+    String _location;
     transferEncoding_t _transferEncoding = HTTPC_TE_IDENTITY;
     std::unique_ptr<StreamString> _payload;
 };

libraries/ESP8266httpUpdate/src/ESP8266httpUpdate.cpp

@@ -30,12 +30,12 @@ extern "C" uint32_t _SPIFFS_start;
 extern "C" uint32_t _SPIFFS_end;
 
 ESP8266HTTPUpdate::ESP8266HTTPUpdate(void)
-        : _httpClientTimeout(8000), _ledPin(-1)
+        : _httpClientTimeout(8000), _followRedirects(false), _ledPin(-1)
 {
 }
 
 ESP8266HTTPUpdate::ESP8266HTTPUpdate(int httpClientTimeout)
-        : _httpClientTimeout(httpClientTimeout), _ledPin(-1)
+        : _httpClientTimeout(httpClientTimeout), _followRedirects(false), _ledPin(-1)
 {
 }
 
@@ -261,6 +261,7 @@ HTTPUpdateResult ESP8266HTTPUpdate::handleUpdate(HTTPClient& http, const String&
     // use HTTP/1.0 for update since the update handler not support any transfer Encoding
     http.useHTTP10(true);
     http.setTimeout(_httpClientTimeout);
+    http.setFollowRedirects(_followRedirects);
     http.setUserAgent(F("ESP8266-http-Update"));
     http.addHeader(F("x-ESP8266-STA-MAC"), WiFi.macAddress());
     http.addHeader(F("x-ESP8266-AP-MAC"), WiFi.softAPmacAddress());

libraries/ESP8266httpUpdate/src/ESP8266httpUpdate.h

@@ -74,6 +74,11 @@ public:
         _rebootOnUpdate = reboot;
     }
 
+    void followRedirects(bool follow)
+    {
+        _followRedirects = follow;
+    }
+
     void setLedPin(int ledPin = -1, uint8_t ledOn = HIGH)
     {
         _ledPin = ledPin;
@@ -129,6 +134,7 @@ protected:
     bool _rebootOnUpdate = true;
 private:
     int _httpClientTimeout;
+    bool _followRedirects;
 
     int _ledPin;
     uint8_t _ledOn;

tests/device/test_http_client/test_http_client.ino

@@ -65,9 +65,110 @@ TEST_CASE("HTTP GET & POST requests", "[HTTPClient]")
         REQUIRE(httpCode == HTTPC_ERROR_CONNECTION_REFUSED);
         http.end();
     }
+    {
+        // 301 redirect with follow enabled
+        WiFiClient client;
+        HTTPClient http;
+        http.setFollowRedirects(true);
+        String uri = String("/redirect301?host=")+getenv("SERVER_IP");
+        http.begin(client, getenv("SERVER_IP"), 8088, uri.c_str());
+        auto httpCode = http.GET();
+        REQUIRE(httpCode == HTTP_CODE_OK);
+        String payload = http.getString();
+        REQUIRE(payload == "redirect success");
+    }
+    {
+        // 301 redirect with follow disabled
+        WiFiClient client;
+        HTTPClient http;
+        String uri = String("/redirect301?host=")+getenv("SERVER_IP");
+        http.begin(client, getenv("SERVER_IP"), 8088, uri.c_str());
+        auto httpCode = http.GET();
+        REQUIRE(httpCode == 301);
+    }
+    {
+        // 302 redirect with follow enabled
+        WiFiClient client;
+        HTTPClient http;
+        http.setFollowRedirects(true);
+        String uri = String("/redirect302?host=")+getenv("SERVER_IP");
+        http.begin(client, getenv("SERVER_IP"), 8088, uri.c_str());
+        auto httpCode = http.GET();
+        REQUIRE(httpCode == HTTP_CODE_OK);
+        String payload = http.getString();
+        REQUIRE(payload == "redirect success");
+    }
+    {
+        // 302 redirect with follow disabled
+        WiFiClient client;
+        HTTPClient http;
+        String uri = String("/redirect302?host=")+getenv("SERVER_IP");
+        http.begin(client, getenv("SERVER_IP"), 8088, uri.c_str());
+        auto httpCode = http.GET();
+        REQUIRE(httpCode == 302);
+    }
+    {
+        // 307 redirect with follow enabled
+        WiFiClient client;
+        HTTPClient http;
+        http.setFollowRedirects(true);
+        String uri = String("/redirect307?host=")+getenv("SERVER_IP");
+        http.begin(client, getenv("SERVER_IP"), 8088, uri.c_str());
+        auto httpCode = http.GET();
+        REQUIRE(httpCode == HTTP_CODE_OK);
+        String payload = http.getString();
+        REQUIRE(payload == "redirect success");
+    }
+    {
+        // 307 redirect with follow disabled
+        WiFiClient client;
+        HTTPClient http;
+        String uri = String("/redirect307?host=")+getenv("SERVER_IP");
+        http.begin(client, getenv("SERVER_IP"), 8088, uri.c_str());
+        auto httpCode = http.GET();
+        REQUIRE(httpCode == 307);
+    }
+    {
+        // 301 exceeding redirect limit
+        WiFiClient client;
+        HTTPClient http;
+        http.setFollowRedirects(true);
+        http.setRedirectLimit(0);
+        String uri = String("/redirect301?host=")+getenv("SERVER_IP");
+        http.begin(client, getenv("SERVER_IP"), 8088, uri.c_str());
+        auto httpCode = http.GET();
+        REQUIRE(httpCode == 301);
+    }
+    {
+        // POST 303 redirect with follow enabled
+        WiFiClient client;
+        HTTPClient http;
+        http.setFollowRedirects(true);
+        http.begin(client, getenv("SERVER_IP"), 8088, "/redirect303");
+        auto httpCode = http.POST(getenv("SERVER_IP"));
+        REQUIRE(httpCode == HTTP_CODE_OK);
+        String payload = http.getString();
+        REQUIRE(payload == "redirect success");
+    }
+    {
+        // POST 303 redirect with follow disabled
+        WiFiClient client;
+        HTTPClient http;
+        http.begin(client, getenv("SERVER_IP"), 8088, "/redirect303");
+        auto httpCode = http.POST(getenv("SERVER_IP"));
+        REQUIRE(httpCode == 303);
+    }
+    {
+        // 302 redirect with follow disabled
+        WiFiClient client;
+        HTTPClient http;
+        String uri = String("/redirect302?host=")+getenv("SERVER_IP");
+        http.begin(client, getenv("SERVER_IP"), 8088, uri.c_str());
+        auto httpCode = http.GET();
+        REQUIRE(httpCode == 302);
+    }
 }
 
-
 TEST_CASE("HTTPS GET request", "[HTTPClient]")
 {
     //

tests/device/test_http_client/test_http_client.py

@@ -1,5 +1,5 @@
 from mock_decorators import setup, teardown
-from flask import Flask, request
+from flask import Flask, request, redirect
 from threading import Thread
 import urllib2
 import os
@@ -26,6 +26,21 @@ def setup_http_get(e):
     def get_data():
         size = int(request.args['size'])
         return 'a'*size
+    @app.route("/target")
+    def target():
+        return "redirect success"
+    @app.route("/redirect301")
+    def redirect301():
+        return redirect("http://{}:8088/target".format(request.args['host']), code=301)
+    @app.route("/redirect302")
+    def redirect302():
+        return redirect("http://{}:8088/target".format(request.args['host']), code=302)
+    @app.route("/redirect303", methods = ['POST'])
+    def redirect303():
+        return redirect("http://{}:8088/target".format(request.data), code=303)
+    @app.route("/redirect307")
+    def redirect307():
+        return redirect("http://{}:8088/target".format(request.args['host']), code=307)
     def flaskThread():
         app.run(host='0.0.0.0', port=8088)    
     th = Thread(target=flaskThread)
@@ -35,7 +50,7 @@ def setup_http_get(e):
 def teardown_http_get(e):
     response = urllib2.urlopen('http://localhost:8088/shutdown')
     html = response.read()
-    time.sleep(30)
+    time.sleep(1) # avoid address in use error on macOS
 
 
 @setup('HTTPS GET request')