Cleanup base64::encode functions (#6607)

* Cleanup base64::encode functions

The implementation choice here using libb64 is generally good as it
is a relatively fast implementation, however the adaptation to
use PROGMEM for the translation function was a bad choice, as reading
randomly PROGMEM with byte-wide access is very very very slow.

Doing a naive if-snake is between 20% and 55% faster and uses less
flash (about 120 bytes less) and also for reasons I don't understand
8 bytes less data RAM (maybe the removal of static?).

In addition the base64::encode function was allocating for larger
input a huge amount of memory (twice the total size). we can reduce
that by doing a chunk-wise conversation to base64.

* Create authorisation base64 encoded string without newlines

Rather than first creating a string with newlines and then
stripping it away in the fast path of constructing the query,
we can call the right method and trust that the result does
not have newlines anymore.

cores/esp8266/base64.cpp

@@ -24,7 +24,6 @@
 
 #include "Arduino.h"
 extern "C" {
-#include "libb64/cdecode.h"
 #include "libb64/cencode.h"
 }
 #include "base64.h"
@@ -35,14 +34,19 @@ extern "C" {
  * @param length size_t
  * @return String
  */
-String base64::encode(const uint8_t * data, size_t length, bool doNewLines) {
+String base64::encode(const uint8_t * data, size_t length, bool doNewLines)
+{
+    String base64;
+
     // base64 needs more size then the source data, use cencode.h macros
-    size_t size = ((doNewLines ? base64_encode_expected_len(length)
+    size_t size = ((doNewLines ? base64_encode_expected_len( length )
-                               : base64_encode_expected_len_nonewlines(length)) + 1);
+                    : base64_encode_expected_len_nonewlines( length )) + 1);
-    char * buffer = (char *) malloc(size);
+
-    if(buffer) {
+    if (base64.reserve(size))
+    {
+
         base64_encodestate _state;
-        if(doNewLines)
+        if (doNewLines)
         {
             base64_init_encodestate(&_state);
         }
@@ -50,22 +54,23 @@ String base64::encode(const uint8_t * data, size_t length, bool doNewLines) {
         {
             base64_init_encodestate_nonewlines(&_state);
         }
-        int len = base64_encode_block((const char *) &data[0], length, &buffer[0], &_state);
-        len = base64_encode_blockend((buffer + len), &_state);
 
-        String base64 = String(buffer);
+        constexpr size_t BUFSIZE = 48;
-        free(buffer);
+        char buf[BUFSIZE + 1 /* newline */ + 1 /* NUL */];
-        return base64;
+        for (size_t len = 0; len < length; len += BUFSIZE * 3 / 4)
+        {
+            size_t blocklen = base64_encode_block((const char*) data + len,
+                                                  std::min( BUFSIZE * 3 / 4, length - len ), buf, &_state);
+            buf[blocklen] = '\0';
+            base64 += buf;
+        }
+        if (base64_encode_blockend(buf, &_state))
+            base64 += buf;
+    }
+    else
+    {
+        base64 = F("-FAIL-");
     }
-    return String("-FAIL-");
-}
 
-/**
+    return base64;
- * convert input data to base64
- * @param text const String&
- * @return String
- */
-String base64::encode(const String& text, bool doNewLines) {
-    return base64::encode((const uint8_t *) text.c_str(), text.length(), doNewLines);
 }
-

cores/esp8266/base64.h

@@ -25,14 +25,18 @@
 #ifndef CORE_BASE64_H_
 #define CORE_BASE64_H_
 
-class base64 {
+class base64
-    public:
+{
-        // NOTE: The default behaviour of backend (lib64)
+public:
-        // is to add a newline every 72 (encoded) characters output.
+    // NOTE: The default behaviour of backend (lib64)
-        // This may 'break' longer uris and json variables
+    // is to add a newline every 72 (encoded) characters output.
-        static String encode(const uint8_t * data, size_t length, bool doNewLines = true);
+    // This may 'break' longer uris and json variables
-        static String encode(const String& text, bool doNewLines = true);
+    static String encode(const uint8_t * data, size_t length, bool doNewLines = true);
-    private:
+    static String inline encode(const String& text, bool doNewLines = true)
+    {
+        return encode( (const uint8_t *) text.c_str(), text.length(), doNewLines );
+    }
+private:
 };
 
 

cores/esp8266/libb64/cencode.cpp

@@ -5,7 +5,6 @@ This is part of the libb64 project, and has been placed in the public domain.
 For details, see http://sourceforge.net/projects/libb64
 */
 
-#include <pgmspace.h>
 #include "cencode.h"
 
 extern "C" {
@@ -23,10 +22,20 @@ void base64_init_encodestate_nonewlines(base64_encodestate* state_in){
   state_in->stepsnewline = -1;
 }
 
-char base64_encode_value(char value_in){
+char base64_encode_value(const char n) {
-  static const char encoding[] PROGMEM = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+  char r;
-  if (value_in > 63) return '=';
+
-  return pgm_read_byte( &encoding[(int)value_in] );
+  if (n < 26)
+    r = n + 'A';
+  else if (n < 26 + 26)
+    r = n - 26 + 'a';
+  else if (n < 26 + 26 + 10 )
+    r = n - 26 - 26 + '0';
+  else if (n == 62 )
+    r = '+';
+  else
+    r = '/';
+  return r;
 }
 
 int base64_encode_block(const char* plaintext_in, int length_in, char* code_out, base64_encodestate* state_in){

libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp

@@ -319,7 +319,7 @@ bool HTTPClient::beginInternal(const String& __url, const char* expectedProtocol
         // auth info
         String auth = host.substring(0, index);
         host.remove(0, index + 1); // remove auth part including @
-        _base64Authorization = base64::encode(auth);
+        _base64Authorization = base64::encode(auth, false /* doNewLines */);
     }
 
     // get port
@@ -504,7 +504,7 @@ void HTTPClient::setAuthorization(const char * user, const char * password)
         String auth = user;
         auth += ':';
         auth += password;
-        _base64Authorization = base64::encode(auth);
+        _base64Authorization = base64::encode(auth, false /* doNewLines */);
     }
 }
 
@@ -516,6 +516,7 @@ void HTTPClient::setAuthorization(const char * auth)
 {
     if(auth) {
         _base64Authorization = auth;
+        _base64Authorization.replace(String('\n'), emptyString);
     }
 }
 
@@ -1243,7 +1244,6 @@ bool HTTPClient::sendHeader(const char * type)
     }
 
     if(_base64Authorization.length()) {
-        _base64Authorization.replace("\n", "");
         header += F("Authorization: Basic ");
         header += _base64Authorization;
         header += "\r\n";