Stream::send() (#6979)

cores/esp8266/Client.h

@@ -26,15 +26,15 @@
 class Client: public Stream {
 
     public:
-        virtual int connect(IPAddress ip, uint16_t port) =0;
+        virtual int connect(IPAddress ip, uint16_t port) = 0;
-        virtual int connect(const char *host, uint16_t port) =0;
+        virtual int connect(const char *host, uint16_t port) = 0;
-        virtual size_t write(uint8_t) =0;
+        virtual size_t write(uint8_t) override = 0;
-        virtual size_t write(const uint8_t *buf, size_t size) =0;
+        virtual size_t write(const uint8_t *buf, size_t size) override = 0;
-        virtual int available() = 0;
+        virtual int available() override = 0;
-        virtual int read() = 0;
+        virtual int read() override = 0;
-        virtual int read(uint8_t *buf, size_t size) = 0;
+        virtual int read(uint8_t *buf, size_t size) override = 0;
-        virtual int peek() = 0;
+        virtual int peek() override = 0;
-        virtual void flush() = 0;
+        virtual void flush() override = 0;
         virtual void stop() = 0;
         virtual uint8_t connected() = 0;
         virtual operator bool() = 0;

cores/esp8266/FS.cpp

@@ -66,7 +66,7 @@ int File::read() {
     return result;
 }
 
-size_t File::read(uint8_t* buf, size_t size) {
+int File::read(uint8_t* buf, size_t size) {
     if (!_p)
         return 0;
 

cores/esp8266/FS.h

@@ -67,13 +67,14 @@ public:
     size_t readBytes(char *buffer, size_t length) override {
         return read((uint8_t*)buffer, length);
     }
-    size_t read(uint8_t* buf, size_t size);
+    int read(uint8_t* buf, size_t size) override;
     bool seek(uint32_t pos, SeekMode mode);
     bool seek(uint32_t pos) {
         return seek(pos, SeekSet);
     }
     size_t position() const;
     size_t size() const;
+    virtual ssize_t streamRemaining() override { return (ssize_t)size() - (ssize_t)position(); }
     void close();
     operator bool() const;
     const char* name() const;
@@ -84,6 +85,7 @@ public:
     bool isDirectory() const;
 
     // Arduino "class SD" methods for compatibility
+    //TODO use stream::send / check read(buf,size) result
     template<typename T> size_t write(T &src){
       uint8_t obuf[256];
       size_t doneLen = 0;

cores/esp8266/FSImpl.h

@@ -30,7 +30,7 @@ class FileImpl {
 public:
     virtual ~FileImpl() { }
     virtual size_t write(const uint8_t *buf, size_t size) = 0;
-    virtual size_t read(uint8_t* buf, size_t size) = 0;
+    virtual int read(uint8_t* buf, size_t size) = 0;
     virtual void flush() = 0;
     virtual bool seek(uint32_t pos, SeekMode mode) = 0;
     virtual size_t position() const = 0;

cores/esp8266/HardwareSerial.h

@@ -135,16 +135,45 @@ public:
         // return -1 when data is unvailable (arduino api)
         return uart_peek_char(_uart);
     }
+
+    virtual bool hasPeekBufferAPI () const override
+    {
+        return true;
+    }
+
+    // return a pointer to available data buffer (size = available())
+    // semantic forbids any kind of read() before calling peekConsume()
+    const char* peekBuffer () override
+    {
+        return uart_peek_buffer(_uart);
+    }
+
+    // return number of byte accessible by peekBuffer()
+    size_t peekAvailable () override
+    {
+        return uart_peek_available(_uart);
+    }
+
+    // consume bytes after use (see peekBuffer)
+    void peekConsume (size_t consume) override
+    {
+        return uart_peek_consume(_uart, consume);
+    }
+
     int read(void) override
     {
         // return -1 when data is unvailable (arduino api)
         return uart_read_char(_uart);
     }
     // ::read(buffer, size): same as readBytes without timeout
-    size_t read(char* buffer, size_t size)
+    int read(char* buffer, size_t size)
     {
         return uart_read(_uart, buffer, size);
     }
+    int read(uint8_t* buffer, size_t size) override
+    {
+        return uart_read(_uart, (char*)buffer, size);
+    }
     size_t readBytes(char* buffer, size_t size) override;
     size_t readBytes(uint8_t* buffer, size_t size) override
     {

cores/esp8266/Print.cpp

@@ -33,15 +33,7 @@
 
 /* default implementation: may be overridden */
 size_t Print::write(const uint8_t *buffer, size_t size) {
-
+    IAMSLOW();
-#ifdef DEBUG_ESP_CORE
-    static char not_the_best_way [] PROGMEM STORE_ATTR = "Print::write(data,len) should be overridden for better efficiency\r\n";
-    static bool once = false;
-    if (!once) {
-        once = true;
-        os_printf_plus(not_the_best_way);
-    }
-#endif
 
     size_t n = 0;
     while (size--) {

cores/esp8266/Print.h

@@ -111,6 +111,10 @@ class Print {
         size_t println(void);
 
         virtual void flush() { /* Empty implementation for backward compatibility */ }
+
+        // by default write timeout is possible (outgoing data from network,serial..)
+        // (children can override to false (like String))
+        virtual bool outputCanTimeout () { return true; }
 };
 
 template<> size_t Print::printNumber(double number, uint8_t digits);

cores/esp8266/Stream.cpp

@@ -22,6 +22,7 @@
 
 #include <Arduino.h>
 #include <Stream.h>
+
 #define PARSE_TIMEOUT 1000  // default number of milli-seconds to wait
 #define NO_SKIP_CHAR  1  // a magic char not found in a valid ASCII numeric field
 
@@ -210,6 +211,8 @@ float Stream::parseFloat(char skipChar) {
 // the buffer is NOT null terminated.
 //
 size_t Stream::readBytes(char *buffer, size_t length) {
+    IAMSLOW();
+
     size_t count = 0;
     while(count < length) {
         int c = timedRead();
@@ -258,3 +261,20 @@ String Stream::readStringUntil(char terminator) {
     }
     return ret;
 }
+
+// read what can be read, immediate exit on unavailable data
+// prototype similar to Arduino's `int Client::read(buf, len)`
+int Stream::read (uint8_t* buffer, size_t maxLen)
+{
+    IAMSLOW();
+
+    size_t nbread = 0;
+    while (nbread < maxLen && available())
+    {
+        int c = read();
+        if (c == -1)
+            break;
+        buffer[nbread++] = read();
+    }
+    return nbread;
+}

cores/esp8266/Stream.h

@@ -22,10 +22,13 @@
 #ifndef Stream_h
 #define Stream_h
 
+#include <debug.h>
 #include <inttypes.h>
-#include "Print.h"
+#include <Print.h>
+#include <PolledTimeout.h>
+#include <sys/types.h> // ssize_t
 
-// compatability macros for testing
+// compatibility macros for testing
 /*
  #define   getInt()            parseInt()
  #define   getInt(skipChar)    parseInt(skipchar)
@@ -35,6 +38,15 @@
  readBytesBetween( pre_string, terminator, buffer, length)
  */
 
+// Arduino `Client: public Stream` class defines `virtual int read(uint8_t *buf, size_t size) = 0;`
+// This function is now imported into `Stream::` for `Stream::send*()`.
+// Other classes inheriting from `Stream::` and implementing `read(uint8_t *buf, size_t size)`
+// must consequently use `int` as return type, namely Hardware/SoftwareSerial, FileSystems...
+#define STREAM_READ_RETURNS_INT 1
+
+// Stream::send API is present
+#define STREAMSEND_API 1
+
 class Stream: public Print {
     protected:
         unsigned long _timeout = 1000;  // number of milliseconds to wait for the next char before aborting timed read
@@ -53,6 +65,7 @@ class Stream: public Print {
 // parsing methods
 
         void setTimeout(unsigned long timeout);  // sets maximum milliseconds to wait for stream data, default is 1 second
+        unsigned long getTimeout () const { return _timeout; }
 
         bool find(const char *target);   // reads data from the stream until the target string is found
         bool find(uint8_t *target) {
@@ -102,12 +115,114 @@ class Stream: public Print {
         virtual String readString();
         String readStringUntil(char terminator);
 
+        virtual int read (uint8_t* buffer, size_t len);
+        int read (char* buffer, size_t len) { return read((uint8_t*)buffer, len); }
+
+        //////////////////// extension: direct access to input buffer
+        // to provide when possible a pointer to available data for read
+
+        // informs user and ::to*() on effective buffered peek API implementation
+        // by default: not available
+        virtual bool hasPeekBufferAPI () const { return false; }
+
+        // returns number of byte accessible by peekBuffer()
+        virtual size_t peekAvailable () { return 0; }
+
+        // returns a pointer to available data buffer (size = peekAvailable())
+        // semantic forbids any kind of ::read()
+        //     - after calling peekBuffer()
+        //     - and before calling peekConsume()
+        virtual const char* peekBuffer () { return nullptr; }
+
+        // consumes bytes after peekBuffer() use
+        // (then ::read() is allowed)
+        virtual void peekConsume (size_t consume) { (void)consume; }
+
+        // by default read timeout is possible (incoming data from network,serial..)
+        // children can override to false (like String::)
+        virtual bool inputCanTimeout () { return true; }
+
+        // (outputCanTimeout() is defined in Print::)
+
+    ////////////////////////
+        //////////////////// extensions: Streaming streams to streams
+        // Stream::send*()
+        //
+        // Stream::send*() uses 1-copy transfers when peekBuffer API is
+        // available, or makes a regular transfer through a temporary buffer.
+        //
+        // - for efficiency, Stream classes should implement peekAPI when
+        //   possible
+        // - for an efficient timeout management, Print/Stream classes
+        //   should implement {output,input}CanTimeout()
+
+        using oneShotMs = esp8266::polledTimeout::oneShotFastMs;
+        static constexpr int temporaryStackBufferSize = 64;
+
+        // ::send*() methods:
+        // - always stop before timeout when "no-more-input-possible-data"
+        //   or "no-more-output-possible-data" condition is met
+        // - always return number of transfered bytes
+        // When result is 0 or less than requested maxLen, Print::getLastSend()
+        // contains an error reason.
+
+        // transfers already buffered / immediately available data (no timeout)
+        // returns number of transfered bytes
+        size_t sendAvailable (Print* to) { return sendGeneric(to, -1, -1, oneShotMs::alwaysExpired); }
+        size_t sendAvailable (Print& to) { return sendAvailable(&to); }
+
+        // transfers data until timeout
+        // returns number of transfered bytes
+        size_t sendAll (Print* to, const oneShotMs::timeType timeoutMs = oneShotMs::neverExpires) { return sendGeneric(to, -1, -1, timeoutMs); }
+        size_t sendAll (Print& to, const oneShotMs::timeType timeoutMs = oneShotMs::neverExpires) { return sendAll(&to, timeoutMs); }
+
+        // transfers data until a char is encountered (the char is swallowed but not transfered) with timeout
+        // returns number of transfered bytes
+        size_t sendUntil (Print* to, const int readUntilChar, const oneShotMs::timeType timeoutMs = oneShotMs::neverExpires) { return sendGeneric(to, -1, readUntilChar, timeoutMs); }
+        size_t sendUntil (Print& to, const int readUntilChar, const oneShotMs::timeType timeoutMs = oneShotMs::neverExpires) { return sendUntil(&to, readUntilChar, timeoutMs); }
+
+        // transfers data until requested size or timeout
+        // returns number of transfered bytes
+        size_t sendSize (Print* to, const ssize_t maxLen, const oneShotMs::timeType timeoutMs = oneShotMs::neverExpires) { return sendGeneric(to, maxLen, -1, timeoutMs); }
+        size_t sendSize (Print& to, const ssize_t maxLen, const oneShotMs::timeType timeoutMs = oneShotMs::neverExpires) { return sendSize(&to, maxLen, timeoutMs); }
+
+        // remaining size (-1 by default = unknown)
+        virtual ssize_t streamRemaining () { return -1; }
+
+        enum class Report
+        {
+            Success = 0,
+            TimedOut,
+            ReadError,
+            WriteError,
+            ShortOperation,
+        };
+
+        Report getLastSendReport () const { return _sendReport; }
+
     protected:
-        long parseInt(char skipChar); // as above but the given skipChar is ignored
+        size_t sendGeneric (Print* to,
-        // as above but the given skipChar is ignored
+                            const ssize_t len = -1,
+                            const int readUntilChar = -1,
+                            oneShotMs::timeType timeoutMs = oneShotMs::neverExpires /* neverExpires=>getTimeout() */);
+
+        size_t SendGenericPeekBuffer(Print* to, const ssize_t len, const int readUntilChar, const oneShotMs::timeType timeoutMs);
+        size_t SendGenericRegularUntil(Print* to, const ssize_t len, const int readUntilChar, const oneShotMs::timeType timeoutMs);
+        size_t SendGenericRegular(Print* to, const ssize_t len, const oneShotMs::timeType timeoutMs);
+
+        void setReport (Report report) { _sendReport = report; }
+
+    private:
+
+        Report _sendReport = Report::Success;
+
+    //////////////////// end of extensions
+
+    protected:
+        long parseInt(char skipChar); // as parseInt() but the given skipChar is ignored
         // this allows format characters (typically commas) in values to be ignored
 
-        float parseFloat(char skipChar);  // as above but the given skipChar is ignored
+        float parseFloat(char skipChar);  // as parseFloat() but the given skipChar is ignored
 };
 
 #endif

cores/esp8266/StreamDev.h

@@ -0,0 +1,248 @@
+/*
+    StreamDev.h - Stream helpers
+    Copyright (c) 2019 David Gauchard.  All right reserved.
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation; either
+    version 2.1 of the License, or (at your option) any later version.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+*/
+
+#ifndef __STREAMDEV_H
+#define __STREAMDEV_H
+
+#include <limits>
+#include <esp_priv.h>
+#include <StreamString.h>
+
+///////////////////////////////////////////////
+// /dev/null
+// - black hole as output, swallow everything, availableForWrite = infinite
+// - black hole as input, nothing to read, available = 0
+
+class StreamNull: public Stream
+{
+public:
+
+    // Print
+    virtual size_t write(uint8_t) override
+    {
+        return 1;
+    }
+
+    virtual size_t write(const uint8_t* buffer, size_t size) override
+    {
+        (void)buffer;
+        (void)size;
+        return size;
+    }
+
+    virtual int availableForWrite() override
+    {
+        return std::numeric_limits<int16_t>::max();
+    }
+
+    // Stream
+    virtual int available() override
+    {
+        return 0;
+    }
+
+    virtual int read() override
+    {
+        return -1;
+    }
+
+    virtual int peek() override
+    {
+        return -1;
+    }
+
+    virtual size_t readBytes(char* buffer, size_t len) override
+    {
+        (void)buffer;
+        (void)len;
+        return 0;
+    }
+
+    virtual int read(uint8_t* buffer, size_t len) override
+    {
+        (void)buffer;
+        (void)len;
+        return 0;
+    }
+
+    virtual bool outputCanTimeout() override
+    {
+        return false;
+    }
+
+    virtual bool inputCanTimeout() override
+    {
+        return false;
+    }
+
+    virtual ssize_t streamRemaining() override
+    {
+        return 0;
+    }
+};
+
+///////////////////////////////////////////////
+// /dev/zero
+// - black hole as output, swallow everything, availableForWrite = infinite
+// - big bang as input, gives infinity to read, available = infinite
+
+class StreamZero: public StreamNull
+{
+protected:
+
+    char _zero;
+
+public:
+
+    StreamZero(char zero = 0): _zero(zero) { }
+
+    // Stream
+    virtual int available() override
+    {
+        return std::numeric_limits<int16_t>::max();
+    }
+
+    virtual int read() override
+    {
+        return _zero;
+    }
+
+    virtual int peek() override
+    {
+        return _zero;
+    }
+
+    virtual size_t readBytes(char* buffer, size_t len) override
+    {
+        memset(buffer, _zero, len);
+        return len;
+    }
+
+    virtual int read(uint8_t* buffer, size_t len) override
+    {
+        memset((char*)buffer, _zero, len);
+        return len;
+    }
+
+    virtual ssize_t streamRemaining() override
+    {
+        return std::numeric_limits<int16_t>::max();
+    }
+};
+
+///////////////////////////////////////////////
+// static buffer (in flash or ram)
+// - black hole as output, swallow everything, availableForWrite = infinite
+// - Stream buffer out as input, resettable
+
+class StreamConstPtr: public StreamNull
+{
+protected:
+    const char* _buffer;
+    size_t _size;
+    bool _byteAddressable;
+    size_t _peekPointer = 0;
+
+public:
+    StreamConstPtr(const String& string): _buffer(string.c_str()), _size(string.length()), _byteAddressable(true) { }
+    StreamConstPtr(const char* buffer, size_t size): _buffer(buffer), _size(size), _byteAddressable(__byteAddressable(buffer)) { }
+    StreamConstPtr(const uint8_t* buffer, size_t size): _buffer((const char*)buffer), _size(size), _byteAddressable(__byteAddressable(buffer)) { }
+    StreamConstPtr(const __FlashStringHelper* buffer, size_t size): _buffer(reinterpret_cast<const char*>(buffer)), _size(size), _byteAddressable(false) { }
+    StreamConstPtr(const __FlashStringHelper* text): _buffer(reinterpret_cast<const char*>(text)), _size(strlen_P((PGM_P)text)), _byteAddressable(false) { }
+
+    void resetPointer(int pointer = 0)
+    {
+        _peekPointer = pointer;
+    }
+
+    // Stream
+    virtual int available() override
+    {
+        return peekAvailable();
+    }
+
+    virtual int read() override
+    {
+        return _peekPointer < _size ? _buffer[_peekPointer++] : -1;
+    }
+
+    virtual int peek() override
+    {
+        return _peekPointer < _size ? _buffer[_peekPointer] : -1;
+    }
+
+    virtual size_t readBytes(char* buffer, size_t len) override
+    {
+        if (_peekPointer >= _size)
+        {
+            return 0;
+        }
+        size_t cpylen = std::min(_size - _peekPointer, len);
+        memcpy_P(buffer, _buffer + _peekPointer, cpylen); // whether byte adressible is true
+        _peekPointer += cpylen;
+        return cpylen;
+    }
+
+    virtual int read(uint8_t* buffer, size_t len) override
+    {
+        return readBytes((char*)buffer, len);
+    }
+
+    virtual ssize_t streamRemaining() override
+    {
+        return _size;
+    }
+
+    // peekBuffer
+    virtual bool hasPeekBufferAPI() const override
+    {
+        return _byteAddressable;
+    }
+
+    virtual size_t peekAvailable() override
+    {
+        return _peekPointer < _size ? _size - _peekPointer : 0;
+    }
+
+    virtual const char* peekBuffer() override
+    {
+        return _peekPointer < _size ? _buffer + _peekPointer : nullptr;
+    }
+
+    virtual void peekConsume(size_t consume) override
+    {
+        _peekPointer += consume;
+    }
+};
+
+///////////////////////////////////////////////
+
+Stream& operator << (Stream& out, String& string);
+Stream& operator << (Stream& out, Stream& stream);
+Stream& operator << (Stream& out, StreamString& stream);
+Stream& operator << (Stream& out, const char* text);
+Stream& operator << (Stream& out, const __FlashStringHelper* text);
+
+///////////////////////////////////////////////
+
+#if !defined(NO_GLOBAL_INSTANCES) && !defined(NO_GLOBAL_STREAMDEV)
+extern StreamNull devnull;
+#endif
+
+#endif // __STREAMDEV_H

cores/esp8266/StreamSend.cpp

@@ -0,0 +1,383 @@
+/*
+    StreamDev.cpp - 1-copy transfer related methods
+    Copyright (c) 2019 David Gauchard.  All right reserved.
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation; either
+    version 2.1 of the License, or (at your option) any later version.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
+    parsing functions based on TextFinder library by Michael Margolis
+*/
+
+
+#include <Arduino.h>
+#include <StreamDev.h>
+
+size_t Stream::sendGeneric(Print* to,
+                           const ssize_t len,
+                           const int readUntilChar,
+                           const esp8266::polledTimeout::oneShotFastMs::timeType timeoutMs)
+{
+    setReport(Report::Success);
+
+    if (len == 0)
+    {
+        return 0;    // conveniently avoids timeout for no requested data
+    }
+
+    // There are two timeouts:
+    // - read (network, serial, ...)
+    // - write (network, serial, ...)
+    // However
+    // - getTimeout() is for reading only
+    // - there is no getOutputTimeout() api
+    // So we use getTimeout() for both,
+    // (also when inputCanTimeout() is false)
+
+    if (hasPeekBufferAPI())
+    {
+        return SendGenericPeekBuffer(to, len, readUntilChar, timeoutMs);
+    }
+
+    if (readUntilChar >= 0)
+    {
+        return SendGenericRegularUntil(to, len, readUntilChar, timeoutMs);
+    }
+
+    return SendGenericRegular(to, len, timeoutMs);
+}
+
+
+size_t Stream::SendGenericPeekBuffer(Print* to, const ssize_t len, const int readUntilChar, const esp8266::polledTimeout::oneShotFastMs::timeType timeoutMs)
+{
+    // "neverExpires (default, impossible)" is translated to default timeout
+    esp8266::polledTimeout::oneShotFastMs timedOut(timeoutMs >= esp8266::polledTimeout::oneShotFastMs::neverExpires ? getTimeout() : timeoutMs);
+    // len==-1 => maxLen=0 <=> until starvation
+    const size_t maxLen = std::max((ssize_t)0, len);
+    size_t written = 0;
+
+    while (!maxLen || written < maxLen)
+    {
+        size_t avpk = peekAvailable();
+        if (avpk == 0 && !inputCanTimeout())
+        {
+            // no more data to read, ever
+            break;
+        }
+
+        size_t w = to->availableForWrite();
+        if (w == 0 && !to->outputCanTimeout())
+        {
+            // no more data can be written, ever
+            break;
+        }
+
+        w = std::min(w, avpk);
+        if (maxLen)
+        {
+            w = std::min(w, maxLen - written);
+        }
+        if (w)
+        {
+            const char* directbuf = peekBuffer();
+            bool foundChar = false;
+            if (readUntilChar >= 0)
+            {
+                const char* last = (const char*)memchr(directbuf, readUntilChar, w);
+                if (last)
+                {
+                    w = std::min((size_t)(last - directbuf), w);
+                    foundChar = true;
+                }
+            }
+            if (w && ((w = to->write(directbuf, w))))
+            {
+                peekConsume(w);
+                written += w;
+                if (maxLen)
+                {
+                    timedOut.reset();
+                }
+            }
+            if (foundChar)
+            {
+                peekConsume(1);
+                break;
+            }
+        }
+
+        if (!w && !maxLen && readUntilChar < 0)
+        {
+            // nothing has been transferred and no specific condition is requested
+            break;
+        }
+
+        if (timedOut)
+        {
+            // either (maxLen>0) nothing has been transferred for too long
+            // or readUntilChar >= 0 but char is not encountered for too long
+            // or (maxLen=0) too much time has been spent here
+            break;
+        }
+
+        optimistic_yield(1000);
+    }
+
+    if (getLastSendReport() == Report::Success && maxLen > 0)
+    {
+        if (timeoutMs && timedOut)
+        {
+            setReport(Report::TimedOut);
+        }
+        else if ((ssize_t)written != len)
+        {
+            // This is happening when source cannot timeout (ex: a String)
+            // but has not enough data, or a dest has closed or cannot
+            // timeout but is too small (String, buffer...)
+            //
+            // Mark it as an error because user usually wants to get what is
+            // asked for.
+            setReport(Report::ShortOperation);
+        }
+    }
+
+    return written;
+}
+
+size_t Stream::SendGenericRegularUntil(Print* to, const ssize_t len, const int readUntilChar, const esp8266::polledTimeout::oneShotFastMs::timeType timeoutMs)
+{
+    // regular Stream API
+    // no other choice than reading byte by byte
+
+    // "neverExpires (default, impossible)" is translated to default timeout
+    esp8266::polledTimeout::oneShotFastMs timedOut(timeoutMs >= esp8266::polledTimeout::oneShotFastMs::neverExpires ? getTimeout() : timeoutMs);
+    // len==-1 => maxLen=0 <=> until starvation
+    const size_t maxLen = std::max((ssize_t)0, len);
+    size_t written = 0;
+
+    while (!maxLen || written < maxLen)
+    {
+        size_t avr = available();
+        if (avr == 0 && !inputCanTimeout())
+        {
+            // no more data to read, ever
+            break;
+        }
+
+        size_t w = to->availableForWrite();
+        if (w == 0 && !to->outputCanTimeout())
+        {
+            // no more data can be written, ever
+            break;
+        }
+
+        int c = read();
+        if (c != -1)
+        {
+            if (c == readUntilChar)
+            {
+                break;
+            }
+            w = to->write(c);
+            if (w != 1)
+            {
+                setReport(Report::WriteError);
+                break;
+            }
+            written += 1;
+            if (maxLen)
+            {
+                timedOut.reset();
+            }
+        }
+
+        if (!w && !maxLen && readUntilChar < 0)
+        {
+            // nothing has been transferred and no specific condition is requested
+            break;
+        }
+
+        if (timedOut)
+        {
+            // either (maxLen>0) nothing has been transferred for too long
+            // or readUntilChar >= 0 but char is not encountered for too long
+            // or (maxLen=0) too much time has been spent here
+            break;
+        }
+
+        optimistic_yield(1000);
+    }
+
+    if (getLastSendReport() == Report::Success && maxLen > 0)
+    {
+        if (timeoutMs && timedOut)
+        {
+            setReport(Report::TimedOut);
+        }
+        else if ((ssize_t)written != len)
+        {
+            // This is happening when source cannot timeout (ex: a String)
+            // but has not enough data, or a dest has closed or cannot
+            // timeout but is too small (String, buffer...)
+            //
+            // Mark it as an error because user usually wants to get what is
+            // asked for.
+            setReport(Report::ShortOperation);
+        }
+    }
+
+    return written;
+}
+
+size_t Stream::SendGenericRegular(Print* to, const ssize_t len, const esp8266::polledTimeout::oneShotFastMs::timeType timeoutMs)
+{
+    // regular Stream API
+    // use an intermediary buffer
+
+    // "neverExpires (default, impossible)" is translated to default timeout
+    esp8266::polledTimeout::oneShotFastMs timedOut(timeoutMs >= esp8266::polledTimeout::oneShotFastMs::neverExpires ? getTimeout() : timeoutMs);
+    // len==-1 => maxLen=0 <=> until starvation
+    const size_t maxLen = std::max((ssize_t)0, len);
+    size_t written = 0;
+
+    while (!maxLen || written < maxLen)
+    {
+        size_t avr = available();
+        if (avr == 0 && !inputCanTimeout())
+        {
+            // no more data to read, ever
+            break;
+        }
+
+        size_t w = to->availableForWrite();
+        if (w == 0 && !to->outputCanTimeout())
+            // no more data can be written, ever
+        {
+            break;
+        }
+
+        w = std::min(w, avr);
+        if (maxLen)
+        {
+            w = std::min(w, maxLen - written);
+        }
+        w = std::min(w, (decltype(w))temporaryStackBufferSize);
+        if (w)
+        {
+            char temp[w];
+            ssize_t r = read(temp, w);
+            if (r < 0)
+            {
+                setReport(Report::ReadError);
+                break;
+            }
+            w = to->write(temp, r);
+            written += w;
+            if ((size_t)r != w)
+            {
+                setReport(Report::WriteError);
+                break;
+            }
+            if (maxLen && w)
+            {
+                timedOut.reset();
+            }
+        }
+
+        if (!w && !maxLen)
+        {
+            // nothing has been transferred and no specific condition is requested
+            break;
+        }
+
+        if (timedOut)
+        {
+            // either (maxLen>0) nothing has been transferred for too long
+            // or readUntilChar >= 0 but char is not encountered for too long
+            // or (maxLen=0) too much time has been spent here
+            break;
+        }
+
+        optimistic_yield(1000);
+    }
+
+    if (getLastSendReport() == Report::Success && maxLen > 0)
+    {
+        if (timeoutMs && timedOut)
+        {
+            setReport(Report::TimedOut);
+        }
+        else if ((ssize_t)written != len)
+        {
+            // This is happening when source cannot timeout (ex: a String)
+            // but has not enough data, or a dest has closed or cannot
+            // timeout but is too small (String, buffer...)
+            //
+            // Mark it as an error because user usually wants to get what is
+            // asked for.
+            setReport(Report::ShortOperation);
+        }
+    }
+
+    return written;
+}
+
+Stream& operator << (Stream& out, String& string)
+{
+    StreamConstPtr(string).sendAll(out);
+    return out;
+}
+
+Stream& operator << (Stream& out, StreamString& stream)
+{
+    stream.sendAll(out);
+    return out;
+}
+
+Stream& operator << (Stream& out, Stream& stream)
+{
+    if (stream.streamRemaining() < 0)
+    {
+        if (stream.inputCanTimeout())
+        {
+            // restrict with only what's buffered on input
+            stream.sendAvailable(out);
+        }
+        else
+        {
+            // take all what is in input
+            stream.sendAll(out);
+        }
+    }
+    else
+    {
+        stream.sendSize(out, stream.streamRemaining());
+    }
+    return out;
+}
+
+Stream& operator << (Stream& out, const char* text)
+{
+    StreamConstPtr(text).sendAll(out);
+    return out;
+}
+
+Stream& operator << (Stream& out, const __FlashStringHelper* text)
+{
+    StreamConstPtr(text).sendAll(out);
+    return out;
+}
+
+#if !defined(NO_GLOBAL_INSTANCES) && !defined(NO_GLOBAL_STREAMDEV)
+StreamNull devnull;
+#endif

cores/esp8266/StreamString.cpp

@@ -1,68 +0,0 @@
-/**
- StreamString.cpp
-
- Copyright (c) 2015 Markus Sattler. All rights reserved.
- This file is part of the esp8266 core for Arduino environment.
-
- This library is free software; you can redistribute it and/or
- modify it under the terms of the GNU Lesser General Public
- License as published by the Free Software Foundation; either
- version 2.1 of the License, or (at your option) any later version.
-
- This library is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- Lesser General Public License for more details.
-
- You should have received a copy of the GNU Lesser General Public
- License along with this library; if not, write to the Free Software
- Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
-
- */
-
-#include <Arduino.h>
-#include "StreamString.h"
-
-size_t StreamString::write(const uint8_t *data, size_t size) {
-    if(size && data) {
-        const unsigned int newlen = length() + size;
-        if(reserve(newlen + 1)) {
-            memcpy((void *) (wbuffer() + len()), (const void *) data, size);
-            setLen(newlen);
-            *(wbuffer() + newlen) = 0x00; // add null for string end
-            return size;
-        }
-        DEBUGV(":stream2string: OOM (%d->%d)\n", length(), newlen+1);
-    }
-    return 0;
-}
-
-size_t StreamString::write(uint8_t data) {
-    return concat((char) data);
-}
-
-int StreamString::available() {
-    return length();
-}
-
-int StreamString::read() {
-    if(length()) {
-        char c = charAt(0);
-        remove(0, 1);
-        return c;
-
-    }
-    return -1;
-}
-
-int StreamString::peek() {
-    if(length()) {
-        char c = charAt(0);
-        return c;
-    }
-    return -1;
-}
-
-void StreamString::flush() {
-}
-

cores/esp8266/StreamString.h

@@ -1,39 +1,293 @@
 /**
- StreamString.h
+    StreamString.h
 
- Copyright (c) 2015 Markus Sattler. All rights reserved.
+    Copyright (c) 2020 D. Gauchard. All rights reserved.
- This file is part of the esp8266 core for Arduino environment.
+    This file is part of the esp8266 core for Arduino environment.
 
- This library is free software; you can redistribute it and/or
+    This library is free software; you can redistribute it and/or
- modify it under the terms of the GNU Lesser General Public
+    modify it under the terms of the GNU Lesser General Public
- License as published by the Free Software Foundation; either
+    License as published by the Free Software Foundation; either
- version 2.1 of the License, or (at your option) any later version.
+    version 2.1 of the License, or (at your option) any later version.
 
- This library is distributed in the hope that it will be useful,
+    This library is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- Lesser General Public License for more details.
+    Lesser General Public License for more details.
 
- You should have received a copy of the GNU Lesser General Public
+    You should have received a copy of the GNU Lesser General Public
- License along with this library; if not, write to the Free Software
+    License along with this library; if not, write to the Free Software
- Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 
 */
 
-#ifndef STREAMSTRING_H_
+#ifndef __STREAMSTRING_H
-#define STREAMSTRING_H_
+#define __STREAMSTRING_H
 
+#include <limits>
+#include "WString.h"
 
-class StreamString: public Stream, public String {
+///////////////////////////////////////////////////////////////
+// S2Stream points to a String and makes it a Stream
+// (it is also the helper for StreamString)
+
+class S2Stream: public Stream
+{
 public:
-    size_t write(const uint8_t *buffer, size_t size) override;
-    size_t write(uint8_t data) override;
 
-    int available() override;
+    S2Stream(String& string, int peekPointer = -1):
-    int read() override;
+        string(&string), peekPointer(peekPointer)
-    int peek() override;
+    {
-    void flush() override;
+    }
+
+    S2Stream(String* string, int peekPointer = -1):
+        string(string), peekPointer(peekPointer)
+    {
+    }
+
+    virtual int available() override
+    {
+        return string->length();
+    }
+
+    virtual int availableForWrite() override
+    {
+        return std::numeric_limits<int16_t>::max();
+    }
+
+    virtual int read() override
+    {
+        if (peekPointer < 0)
+        {
+            // consume chars
+            if (string->length())
+            {
+                char c = string->charAt(0);
+                string->remove(0, 1);
+                return c;
+            }
+        }
+        else if (peekPointer < (int)string->length())
+        {
+            // return pointed and move pointer
+            return string->charAt(peekPointer++);
+        }
+
+        // everything is read
+        return -1;
+    }
+
+    virtual size_t write(uint8_t data) override
+    {
+        return string->concat((char)data);
+    }
+
+    virtual int read(uint8_t* buffer, size_t len) override
+    {
+        if (peekPointer < 0)
+        {
+            // string will be consumed
+            size_t l = std::min(len, (size_t)string->length());
+            memcpy(buffer, string->c_str(), l);
+            string->remove(0, l);
+            return l;
+        }
+
+        if (peekPointer >= (int)string->length())
+        {
+            return 0;
+        }
+
+        // only the pointer is moved
+        size_t l = std::min(len, (size_t)(string->length() - peekPointer));
+        memcpy(buffer, string->c_str() + peekPointer, l);
+        peekPointer += l;
+        return l;
+    }
+
+    virtual size_t write(const uint8_t* buffer, size_t len) override
+    {
+        return string->concat((const char*)buffer, len) ? len : 0;
+    }
+
+    virtual int peek() override
+    {
+        if (peekPointer < 0)
+        {
+            if (string->length())
+            {
+                return string->charAt(0);
+            }
+        }
+        else if (peekPointer < (int)string->length())
+        {
+            return string->charAt(peekPointer);
+        }
+
+        return -1;
+    }
+
+    virtual void flush() override
+    {
+        // nothing to do
+    }
+
+    virtual bool inputCanTimeout() override
+    {
+        return false;
+    }
+
+    virtual bool outputCanTimeout() override
+    {
+        return false;
+    }
+
+    //// Stream's peekBufferAPI
+
+    virtual bool hasPeekBufferAPI() const override
+    {
+        return true;
+    }
+
+    virtual size_t peekAvailable()
+    {
+        if (peekPointer < 0)
+        {
+            return string->length();
+        }
+        return string->length() - peekPointer;
+    }
+
+    virtual const char* peekBuffer() override
+    {
+        if (peekPointer < 0)
+        {
+            return string->c_str();
+        }
+        if (peekPointer < (int)string->length())
+        {
+            return string->c_str() + peekPointer;
+        }
+        return nullptr;
+    }
+
+    virtual void peekConsume(size_t consume) override
+    {
+        if (peekPointer < 0)
+        {
+            // string is really consumed
+            string->remove(0, consume);
+        }
+        else
+        {
+            // only the pointer is moved
+            peekPointer = std::min((size_t)string->length(), peekPointer + consume);
+        }
+    }
+
+    virtual ssize_t streamRemaining() override
+    {
+        return peekPointer < 0 ? string->length() : string->length() - peekPointer;
+    }
+
+    // calling setConsume() will consume bytes as the stream is read
+    // (enabled by default)
+    void setConsume()
+    {
+        peekPointer = -1;
+    }
+
+    // Reading this stream will mark the string as read without consuming
+    // (not enabled by default)
+    // Calling resetPointer() resets the read state and allows rereading.
+    void resetPointer(int pointer = 0)
+    {
+        peekPointer = pointer;
+    }
+
+protected:
+
+    String* string;
+    int peekPointer; // -1:String is consumed / >=0:resettable pointer
 };
 
 
-#endif /* STREAMSTRING_H_ */
+// StreamString is a S2Stream holding the String
+
+class StreamString: public String, public S2Stream
+{
+protected:
+
+    void resetpp()
+    {
+        if (peekPointer > 0)
+        {
+            peekPointer = 0;
+        }
+    }
+
+public:
+
+    StreamString(StreamString&& bro): String(bro), S2Stream(this) { }
+    StreamString(const StreamString& bro): String(bro), S2Stream(this) { }
+
+    // duplicate String contructors and operator=:
+
+    StreamString(const char* text = nullptr): String(text), S2Stream(this) { }
+    StreamString(const String& string): String(string), S2Stream(this) { }
+    StreamString(const __FlashStringHelper *str): String(str), S2Stream(this) { }
+    StreamString(String&& string): String(string), S2Stream(this) { }
+    StreamString(StringSumHelper&& sum): String(sum), S2Stream(this) { }
+
+    explicit StreamString(char c): String(c), S2Stream(this) { }
+    explicit StreamString(unsigned char c, unsigned char base = 10): String(c, base), S2Stream(this) { }
+    explicit StreamString(int i, unsigned char base = 10): String(i, base), S2Stream(this) { }
+    explicit StreamString(unsigned int i, unsigned char base = 10): String(i, base), S2Stream(this) { }
+    explicit StreamString(long l, unsigned char base = 10): String(l, base), S2Stream(this) { }
+    explicit StreamString(unsigned long l, unsigned char base = 10): String(l, base), S2Stream(this) { }
+    explicit StreamString(float f, unsigned char decimalPlaces = 2): String(f, decimalPlaces), S2Stream(this) { }
+    explicit StreamString(double d, unsigned char decimalPlaces = 2): String(d, decimalPlaces), S2Stream(this) { }
+
+    StreamString& operator= (const StreamString& rhs)
+    {
+        String::operator=(rhs);
+        resetpp();
+        return *this;
+    }
+
+    StreamString& operator= (const String& rhs)
+    {
+        String::operator=(rhs);
+        resetpp();
+        return *this;
+    }
+
+    StreamString& operator= (const char* cstr)
+    {
+        String::operator=(cstr);
+        resetpp();
+        return *this;
+    }
+
+    StreamString& operator= (const __FlashStringHelper* str)
+    {
+        String::operator=(str);
+        resetpp();
+        return *this;
+    }
+
+    StreamString& operator= (String&& rval)
+    {
+        String::operator=(rval);
+        resetpp();
+        return *this;
+    }
+
+    StreamString& operator= (StringSumHelper&& rval)
+    {
+        String::operator=(rval);
+        resetpp();
+        return *this;
+    }
+};
+
+#endif // __STREAMSTRING_H

cores/esp8266/debug.cpp

@@ -22,6 +22,13 @@
 #include "debug.h"
 #include "osapi.h"
 
+#ifdef DEBUG_ESP_CORE
+void __iamslow(const char* what)
+{
+    DEBUGV("%s should be overridden for better efficiency\r\n", what);
+}
+#endif
+
 IRAM_ATTR
 void hexdump(const void *mem, uint32_t len, uint8_t cols)
 {

cores/esp8266/debug.h

@@ -26,6 +26,20 @@ void __unhandled_exception(const char *str) __attribute__((noreturn));
 void __panic_func(const char* file, int line, const char* func) __attribute__((noreturn));
 #define panic() __panic_func(PSTR(__FILE__), __LINE__, __func__)
 
+#ifdef DEBUG_ESP_CORE
+extern void __iamslow(const char* what);
+#define IAMSLOW() \
+    do { \
+        static bool once = false; \
+        if (!once) { \
+            once = true; \
+            __iamslow((PGM_P)FPSTR(__FUNCTION__)); \
+        } \
+    } while (0)
+#else
+#define IAMSLOW() do { (void)0; } while (0)
+#endif
+
 #ifdef __cplusplus
 }
 #endif

cores/esp8266/esp_priv.h

@@ -0,0 +1,44 @@
+/*
+    esp_priv.h - private esp8266 helpers
+    Copyright (c) 2020 esp8266/Arduino community.  All right reserved.
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation; either
+    version 2.1 of the License, or (at your option) any later version.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+*/
+
+
+#ifndef __ESP_PRIV
+#define __ESP_PRIV
+
+#if defined(CORE_MOCK)
+
+constexpr bool __byteAddressable(const void* addr)
+{
+    (void)addr;
+    return true;
+}
+
+#else // on hardware
+
+#include <sys/config.h>
+
+// returns true when addr can be used without "pgm_" functions or non32xfer service
+constexpr bool __byteAddressable(const void* addr)
+{
+    return addr < (const void*)(XCHAL_DATARAM0_VADDR + XCHAL_DATARAM0_SIZE);
+}
+
+#endif // on hardware
+
+#endif // __ESP_PRIV

cores/esp8266/spiffs_api.h

@@ -388,10 +388,10 @@ public:
         return result;
     }
 
-    size_t read(uint8_t* buf, size_t size) override
+    int read(uint8_t* buf, size_t size) override
     {
         CHECKFD();
-        auto result = SPIFFS_read(_fs->getFs(), _fd, (void*) buf, size);
+        int result = SPIFFS_read(_fs->getFs(), _fd, (void*) buf, size);
         if (result < 0) {
             DEBUGV("SPIFFS_read rc=%d\r\n", result);
             return 0;

cores/esp8266/uart.cpp

@@ -240,6 +240,41 @@ uart_peek_char(uart_t* uart)
     return ret;
 }
 
+// return number of byte accessible by uart_peek_buffer()
+size_t uart_peek_available (uart_t* uart)
+{
+    // path for further optimization:
+    // - return already copied buffer pointer (= older data)
+    // - or return fifo when buffer is empty but then any move from fifo to
+    //   buffer should be blocked until peek_consume is called
+
+    ETS_UART_INTR_DISABLE();
+    uart_rx_copy_fifo_to_buffer_unsafe(uart);
+    auto rpos = uart->rx_buffer->rpos;
+    auto wpos = uart->rx_buffer->wpos;
+    ETS_UART_INTR_ENABLE();
+    if(wpos < rpos)
+        return uart->rx_buffer->size - rpos;
+    return wpos - rpos;
+}
+
+// return a pointer to available data buffer (size = available())
+// semantic forbids any kind of read() between peekBuffer() and peekConsume()
+const char* uart_peek_buffer (uart_t* uart)
+{
+    return (const char*)&uart->rx_buffer->buffer[uart->rx_buffer->rpos];
+}
+
+// consume bytes after use (see uart_peek_buffer)
+void uart_peek_consume (uart_t* uart, size_t consume)
+{
+    ETS_UART_INTR_DISABLE();
+    uart->rx_buffer->rpos += consume;
+    if (uart->rx_buffer->rpos >= uart->rx_buffer->size)
+        uart->rx_buffer->rpos -= uart->rx_buffer->size;
+    ETS_UART_INTR_ENABLE();
+}
+
 int
 uart_read_char(uart_t* uart)
 {

cores/esp8266/uart.h

@@ -147,6 +147,16 @@ int uart_get_debug();
 void uart_start_detect_baudrate(int uart_nr);
 int uart_detect_baudrate(int uart_nr);
 
+// return number of byte accessible by peekBuffer()
+size_t uart_peek_available (uart_t* uart);
+
+// return a pointer to available data buffer (size = available())
+// semantic forbids any kind of read() before calling peekConsume()
+const char* uart_peek_buffer (uart_t* uart);
+
+// consume bytes after use (see peekBuffer)
+void uart_peek_consume (uart_t* uart, size_t consume);
+
 uint8_t uart_get_bit_length(const int uart_nr);
 
 #if defined (__cplusplus)

doc/reference.rst

@@ -326,3 +326,196 @@ C++
     This assures correct behavior, including handling of all subobjects, which guarantees stability.
 
   History: `#6269 <https://github.com/esp8266/Arduino/issues/6269>`__ `#6309 <https://github.com/esp8266/Arduino/pull/6309>`__ `#6312 <https://github.com/esp8266/Arduino/pull/6312>`__
+
+Streams
+-------
+
+Arduino API
+
+  Stream is one of the core classes in the Arduino API.  Wire, serial, network and
+  filesystems are streams, from which data are read or written.
+
+  Making a transfer with streams is quite common, like for example the
+  historical WiFiSerial sketch:
+
+  .. code:: cpp
+
+    //check clients for data
+    //get data from the telnet client and push it to the UART
+    while (serverClient.available()) {
+      Serial.write(serverClient.read());
+    }
+
+    //check UART for data
+    if (Serial.available()) {
+      size_t len = Serial.available();
+      uint8_t sbuf[len];
+      Serial.readBytes(sbuf, len);
+      //push UART data to all connected telnet clients
+      if (serverClient && serverClient.connected()) {
+        serverClient.write(sbuf, len);
+      }
+    }
+
+  One will notice that in the network to serial direction, data are transfered
+  byte by byte while data are available.  In the other direction, a temporary
+  buffer is created on stack, filled with available serial data, then
+  transferred to network.
+
+  The ``readBytes(buffer, length)`` method includes a timeout to ensure that
+  all required bytes are received.  The ``write(buffer, length)`` (inherited
+  from ``Print::``) function is also usually blocking until the full buffer is
+  transmitted. Both functions return the number of transmitted bytes.
+
+  That's the way the Stream class works and is commonly used.
+
+  Classes derived from ``Stream::`` also usually introduce the ``read(buffer,
+  len)`` method, which is similar to ``readBytes(buffer, len)`` without
+  timeout: the returned value can be less than the requested size, so special
+  care must be taken with this function, introduced in the Arduino
+  ``Client::`` class (cf. AVR reference implementation).
+  This function has also been introduced in other classes
+  that don't derive from ``Client::``, e.g.  ``HardwareSerial::``.
+
+Stream extensions
+
+  Stream extensions are designed to be compatible with Arduino API, and
+  offer additional methods to make transfers more efficient and easier to
+  use.
+
+  The serial to network transfer above can be written like this:
+
+  .. code:: cpp
+
+    serverClient.sendAvailable(Serial); // chunk by chunk
+    Serial.sendAvailable(serverClient); // chunk by chunk
+
+  An echo service can be written like this:
+
+  .. code:: cpp
+
+    serverClient.sendAvailable(serverClient); // tcp echo service
+
+    Serial.sendAvailable(Serial);             // serial software loopback
+
+  Beside reducing coding time, these methods optimize transfers by avoiding
+  buffer copies when possible.
+
+  - User facing API: ``Stream::send()``
+
+    The goal of streams is to transfer data between producers and consumers,
+    like the telnet/serial example above.  Four methods are provided, all of
+    them return the number of transmitted bytes:
+
+    - ``Stream::sendSize(dest, size [, timeout])``
+
+      This method waits up to the given or default timeout to transfer
+      ``size`` bytes to the the ``dest`` Stream.
+
+    - ``Stream::sendUntil(dest, delim [, timeout])``
+
+      This method waits up to the given or default timeout to transfer data
+      until the character ``delim`` is met.  
+      Note: The delimiter is read but not transferred (like ``readBytesUntil``)
+
+    - ``Stream::sendAvailable(dest)``
+
+      This method transfers all already available data to the destination.
+      There is no timeout and the returned value is 0 when there is nothing
+      to transfer or no room in the destination.
+
+    - ``Stream::sendAll(dest [, timeout])``
+
+      This method waits up to the given or default timeout to transfer all
+      available data.  It is useful when source is able to tell that no more
+      data will be available for this call, or when destination can tell
+      that it will no be able to receive anymore.
+
+      For example, a source String will not grow during the transfer, or a
+      particular network connection supposed to send a fixed amount of data
+      before closing.  ``::sendAll()`` will receive all bytes.  Timeout is
+      useful when destination needs processing time (e.g.  network or serial
+      input buffer full = please wait a bit).
+
+  - String, flash strings helpers
+
+    Two additional classes are provided.
+
+    - ``StreamPtr::`` is designed to hold a constant buffer (in ram or flash).
+
+      With this class, a ``Stream::`` can be made from ``const char*``,
+      ``F("some words in flash")`` or ``PROGMEM`` strings.  This class makes
+      no copy, even with data in flash.  For flash content, byte-by-byte
+      transfers is a consequence when "memcpy_P" cannot be used.  Other
+      contents can be transferred at once when possible.
+
+      .. code:: cpp
+
+        StreamPtr css(F("my long css data")); // CSS data not copied to RAM
+        server.sendAll(css);
+
+    - ``S2Stream::`` is designed to make a ``Stream::`` out of a ``String::`` without copy.
+
+      .. code:: cpp
+
+        String helloString("hello");
+        S2Stream hello(helloString);
+        hello.reset(0);        // prevents ::read() to consume the string
+
+        hello.sendAll(Serial); // shows "hello"
+        hello.sendAll(Serial); // shows nothing, content has already been read
+        hello.reset();         // reset content pointer
+        hello.sendAll(Serial); // shows "hello"
+        hello.reset(3);        // reset content pointer to a specific position
+        hello.sendAll(Serial); // shows "lo"
+
+        hello.setConsume();    // ::read() will consume, this is the default
+        Serial.println(helloString.length()); // shows 5
+        hello.sendAll(Serial);                // shows "hello"
+        Serial.println(helloString.length()); // shows 0, string is consumed
+
+      ``StreamString::`` derives from ``S2Stream``
+
+      .. code:: cpp
+
+        StreamString contentStream;
+        client.sendSize(contentStream, SOME_SIZE); // receives at most SOME_SIZE bytes
+
+        // equivalent to:
+
+        String content;
+        S2Stream contentStream(content);
+        client.sendSize(contentStream, SOME_SIZE); // receives at most SOME_SIZE bytes
+        // content has the data
+
+  - Internal Stream API: ``peekBuffer``
+
+    Here is the method list and their significations.  They are currently
+    implemented in ``HardwareSerial``, ``WiFiClient`` and
+    ``WiFiClientSecure``.
+
+    - ``virtual bool hasPeekBufferAPI ()`` returns ``true`` when the API is present in the class
+
+    - ``virtual size_t peekAvailable ()`` returns the number of reachable bytes
+
+    - ``virtual const char* peekBuffer ()`` returns the pointer to these bytes
+
+      This API requires that any kind of ``"read"`` function must not be called after ``peekBuffer()``
+      and until ``peekConsume()`` is called.
+
+    - ``virtual void peekConsume (size_t consume)`` tells to discard that number of bytes
+
+    - ``virtual bool inputCanTimeout ()``
+
+      A ``StringStream`` will return false. A closed network connection returns false.
+      This function allows ``Stream::sendAll()`` to return earlier.
+
+    - ``virtual bool outputCanTimeout ()``
+
+      A closed network connection returns false.
+      This function allows ``Stream::sendAll()`` to return earlier.
+
+    - ``virtual ssize_t streamRemaining()``
+
+      It returns -1 when stream remaining size is unknown, depending on implementation
+      (string size, file size..).

libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp

@@ -25,9 +25,22 @@
 
 #include "ESP8266HTTPClient.h"
 #include <ESP8266WiFi.h>
-#include <StreamString.h>
+#include <StreamDev.h>
 #include <base64.h>
 
+static int StreamReportToHttpClientReport (Stream::Report streamSendError)
+{
+    switch (streamSendError)
+    {
+    case Stream::Report::TimedOut: return HTTPC_ERROR_READ_TIMEOUT;
+    case Stream::Report::ReadError: return HTTPC_ERROR_NO_STREAM;
+    case Stream::Report::WriteError: return HTTPC_ERROR_STREAM_WRITE;
+    case Stream::Report::ShortOperation: return HTTPC_ERROR_STREAM_WRITE;
+    case Stream::Report::Success: return 0;
+    }
+    return 0; // never reached, keep gcc quiet
+}
+
 /**
  * constructor
  */
@@ -429,24 +442,9 @@ int HTTPClient::sendRequest(const char * type, const uint8_t * payload, size_t s
             return returnError(HTTPC_ERROR_SEND_HEADER_FAILED);
         }
 
-        // send Payload if needed
+        // transfer all of it, with send-timeout
-        if (payload && size > 0) {
+        if (size && StreamConstPtr(payload, size).sendAll(_client) != size)
-            size_t bytesWritten = 0;
+            return returnError(HTTPC_ERROR_SEND_PAYLOAD_FAILED);
-            const uint8_t *p = payload;
-            size_t originalSize = size;
-            while (bytesWritten < originalSize) {
-                int written;
-                int towrite = std::min((int)size, (int)HTTP_TCP_BUFFER_SIZE);
-                written = _client->write(p + bytesWritten, towrite);
-                if (written < 0) {
-                        return returnError(HTTPC_ERROR_SEND_PAYLOAD_FAILED);
-                } else if (written == 0) {
-                        return returnError(HTTPC_ERROR_CONNECTION_LOST);
-                }
-                bytesWritten += written;
-                size -= written;
-            }
-        }
 
         // handle Server Response (Header)
         code = handleHeaderResponse();
@@ -545,111 +543,12 @@ int HTTPClient::sendRequest(const char * type, Stream * stream, size_t size)
         return returnError(HTTPC_ERROR_SEND_HEADER_FAILED);
     }
 
-    int buff_size = HTTP_TCP_BUFFER_SIZE;
+    // transfer all of it, with timeout
-
+    size_t transferred = stream->sendSize(_client, size);
-    int len = size;
+    if (transferred != size)
-    int bytesWritten = 0;
+    {
-
+        DEBUG_HTTPCLIENT("[HTTP-Client][sendRequest] short write, asked for %d but got %d failed.\n", size, transferred);
-    if(len == 0) {
+        return returnError(HTTPC_ERROR_SEND_PAYLOAD_FAILED);
-        len = -1;
-    }
-
-    // if possible create smaller buffer then HTTP_TCP_BUFFER_SIZE
-    if((len > 0) && (len < HTTP_TCP_BUFFER_SIZE)) {
-        buff_size = len;
-    }
-
-    // create buffer for read
-    uint8_t * buff = (uint8_t *) malloc(buff_size);
-
-    if(buff) {
-        // read all data from stream and send it to server
-        while(connected() && (stream->available() > 0) && (len > 0 || len == -1)) {
-
-            // get available data size
-            int sizeAvailable = stream->available();
-
-            if(sizeAvailable) {
-
-                int readBytes = sizeAvailable;
-
-                // read only the asked bytes
-                if(len > 0 && readBytes > len) {
-                    readBytes = len;
-                }
-
-                // not read more the buffer can handle
-                if(readBytes > buff_size) {
-                    readBytes = buff_size;
-                }
-
-                // read data
-                int bytesRead = stream->readBytes(buff, readBytes);
-
-                // write it to Stream
-                int bytesWrite = _client->write((const uint8_t *) buff, bytesRead);
-                bytesWritten += bytesWrite;
-
-                // are all Bytes a writen to stream ?
-                if(bytesWrite != bytesRead) {
-                    DEBUG_HTTPCLIENT("[HTTP-Client][sendRequest] short write, asked for %d but got %d retry...\n", bytesRead, bytesWrite);
-
-                    // check for write error
-                    if(_client->getWriteError()) {
-                        DEBUG_HTTPCLIENT("[HTTP-Client][sendRequest] stream write error %d\n", _client->getWriteError());
-
-                        //reset write error for retry
-                        _client->clearWriteError();
-                    }
-
-                    // some time for the stream
-                    delay(1);
-
-                    int leftBytes = (readBytes - bytesWrite);
-
-                    // retry to send the missed bytes
-                    bytesWrite = _client->write((const uint8_t *) (buff + bytesWrite), leftBytes);
-                    bytesWritten += bytesWrite;
-
-                    if(bytesWrite != leftBytes) {
-                        // failed again
-                        DEBUG_HTTPCLIENT("[HTTP-Client][sendRequest] short write, asked for %d but got %d failed.\n", leftBytes, bytesWrite);
-                        free(buff);
-                        return returnError(HTTPC_ERROR_SEND_PAYLOAD_FAILED);
-                    }
-                }
-
-                // check for write error
-                if(_client->getWriteError()) {
-                    DEBUG_HTTPCLIENT("[HTTP-Client][sendRequest] stream write error %d\n", _client->getWriteError());
-                    free(buff);
-                    return returnError(HTTPC_ERROR_SEND_PAYLOAD_FAILED);
-                }
-
-                // count bytes to read left
-                if(len > 0) {
-                    len -= readBytes;
-                }
-
-                delay(0);
-            } else {
-                delay(1);
-            }
-        }
-
-        free(buff);
-
-        if(size && (int) size != bytesWritten) {
-            DEBUG_HTTPCLIENT("[HTTP-Client][sendRequest] Stream payload bytesWritten %d and size %zd mismatch!.\n", bytesWritten, size);
-            DEBUG_HTTPCLIENT("[HTTP-Client][sendRequest] ERROR SEND PAYLOAD FAILED!");
-            return returnError(HTTPC_ERROR_SEND_PAYLOAD_FAILED);
-        } else {
-            DEBUG_HTTPCLIENT("[HTTP-Client][sendRequest] Stream payload written: %d\n", bytesWritten);
-        }
-
-    } else {
-        DEBUG_HTTPCLIENT("[HTTP-Client][sendRequest] not enough ram! need %d\n", HTTP_TCP_BUFFER_SIZE);
-        return returnError(HTTPC_ERROR_TOO_LESS_RAM);
     }
 
     // handle Server Response (Header)
@@ -725,13 +624,13 @@ int HTTPClient::writeToStream(Stream * stream)
     int ret = 0;
 
     if(_transferEncoding == HTTPC_TE_IDENTITY) {
-        if(len > 0 || len == -1) {
+        // len < 0: transfer all of it, with timeout
-            ret = writeToStreamDataBlock(stream, len);
+        // len >= 0: max:len, with timeout
+        ret = _client->sendSize(stream, len);
 
-            // have we an error?
+        // do we have an error?
-            if(ret < 0) {
+        if(_client->getLastSendReport() != Stream::Report::Success) {
-                return returnError(ret);
+            return returnError(StreamReportToHttpClientReport(_client->getLastSendReport()));
-            }
         }
     } else if(_transferEncoding == HTTPC_TE_CHUNKED) {
         int size = 0;
@@ -754,11 +653,11 @@ int HTTPClient::writeToStream(Stream * stream)
 
             // data left?
             if(len > 0) {
-                int r = writeToStreamDataBlock(stream, len);
+                // read len bytes with timeout
-                if(r < 0) {
+                int r = _client->sendSize(stream, len);
-                    // error in writeToStreamDataBlock
+                if (_client->getLastSendReport() != Stream::Report::Success)
-                    return returnError(r);
+                    // not all data transferred
-                }
+                    return returnError(StreamReportToHttpClientReport(_client->getLastSendReport()));
                 ret += r;
             } else {
 
@@ -948,9 +847,7 @@ bool HTTPClient::connect(void)
 {
     if(_reuse && _canReuse && connected()) {
         DEBUG_HTTPCLIENT("[HTTP-Client] connect: already connected, reusing connection\n");
-        while(_client->available() > 0) {
+        _client->sendAvailable(devnull); // clear _client's output (all of it, no timeout)
-            _client->read();
-        }
         return true;
     }
 
@@ -1032,7 +929,8 @@ bool HTTPClient::sendHeader(const char * type)
 
     DEBUG_HTTPCLIENT("[HTTP-Client] sending request header\n-----\n%s-----\n", header.c_str());
 
-    return (_client->write((const uint8_t *) header.c_str(), header.length()) == header.length());
+    // transfer all of it, with timeout
+    return StreamConstPtr(header).sendAll(_client) == header.length();
 }
 
 /**
@@ -1150,116 +1048,6 @@ int HTTPClient::handleHeaderResponse()
     return HTTPC_ERROR_CONNECTION_LOST;
 }
 
-/**
- * write one Data Block to Stream
- * @param stream Stream *
- * @param size int
- * @return < 0 = error >= 0 = size written
- */
-int HTTPClient::writeToStreamDataBlock(Stream * stream, int size)
-{
-    int buff_size = HTTP_TCP_BUFFER_SIZE;
-    int len = size; // left size to read
-    int bytesWritten = 0;
-
-    // if possible create smaller buffer then HTTP_TCP_BUFFER_SIZE
-    if((len > 0) && (len < HTTP_TCP_BUFFER_SIZE)) {
-        buff_size = len;
-    }
-
-    // create buffer for read
-    uint8_t * buff = (uint8_t *) malloc(buff_size);
-
-    if(!buff) {
-        DEBUG_HTTPCLIENT("[HTTP-Client][writeToStreamDataBlock] not enough ram! need %d\n", HTTP_TCP_BUFFER_SIZE);
-        return HTTPC_ERROR_TOO_LESS_RAM;
-    }
-
-    // read all data from server
-    while(connected() && (len > 0 || len == -1))
-    {
-        int readBytes = len;
-
-        // not read more the buffer can handle
-        if(readBytes > buff_size) {
-            readBytes = buff_size;
-        }
-    
-        // len == -1 or len > what is available, read only what is available
-        int av = _client->available();
-        if (readBytes < 0 || readBytes > av) {
-            readBytes = av;
-        }
-
-        // read data
-        int bytesRead = _client->readBytes(buff, readBytes);
-        if (!bytesRead)
-        {
-            DEBUG_HTTPCLIENT("[HTTP-Client][writeToStreamDataBlock] input stream timeout\n");
-            free(buff);
-            return HTTPC_ERROR_READ_TIMEOUT;
-        }
-
-        // write it to Stream
-        int bytesWrite = stream->write(buff, bytesRead);
-        bytesWritten += bytesWrite;
-
-        // are all Bytes a writen to stream ?
-        if(bytesWrite != bytesRead) {
-            DEBUG_HTTPCLIENT("[HTTP-Client][writeToStream] short write asked for %d but got %d retry...\n", bytesRead, bytesWrite);
-
-            // check for write error
-            if(stream->getWriteError()) {
-                DEBUG_HTTPCLIENT("[HTTP-Client][writeToStreamDataBlock] stream write error %d\n", stream->getWriteError());
-
-                //reset write error for retry
-                stream->clearWriteError();
-            }
-
-            // some time for the stream
-            delay(1);
-
-            int leftBytes = (bytesRead - bytesWrite);
-
-            // retry to send the missed bytes
-            bytesWrite = stream->write((buff + bytesWrite), leftBytes);
-            bytesWritten += bytesWrite;
-
-            if(bytesWrite != leftBytes) {
-                // failed again
-                DEBUG_HTTPCLIENT("[HTTP-Client][writeToStream] short write asked for %d but got %d failed.\n", leftBytes, bytesWrite);
-                free(buff);
-                return HTTPC_ERROR_STREAM_WRITE;
-            }
-        }
-
-        // check for write error
-        if(stream->getWriteError()) {
-            DEBUG_HTTPCLIENT("[HTTP-Client][writeToStreamDataBlock] stream write error %d\n", stream->getWriteError());
-            free(buff);
-            return HTTPC_ERROR_STREAM_WRITE;
-        }
-
-        // count bytes to read left
-        if(len > 0) {
-            len -= bytesRead;
-        }
-
-        delay(0);
-    }
-
-    free(buff);
-
-    DEBUG_HTTPCLIENT("[HTTP-Client][writeToStreamDataBlock] end of chunk or data (transferred: %d).\n", bytesWritten);
-
-    if((size > 0) && (size != bytesWritten)) {
-        DEBUG_HTTPCLIENT("[HTTP-Client][writeToStreamDataBlock] transferred size %d and request size %d mismatch!.\n", bytesWritten, size);
-        return HTTPC_ERROR_STREAM_WRITE;
-    }
-
-    return bytesWritten;
-}
-
 /**
  * called to handle error return, may disconnect the connection if still exists
  * @param error

libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.h

@@ -28,7 +28,7 @@
 
 #include <memory>
 #include <Arduino.h>
-
+#include <StreamString.h>
 #include <WiFiClient.h>
 
 #ifdef DEBUG_ESP_HTTP_CLIENT
@@ -148,8 +148,6 @@ typedef enum {
 class TransportTraits;
 typedef std::unique_ptr<TransportTraits> TransportTraitsPtr;
 
-class StreamString;
-
 class HTTPClient
 {
 public:

libraries/ESP8266WebServer/examples/HelloServer/HelloServer.ino

@@ -115,9 +115,9 @@ void setup(void) {
       // swallow the exact amount matching the full request+content,
       // hence the tcp connection cannot be handled anymore by the
       // webserver.
-#ifdef STREAMTO_API
+#ifdef STREAMSEND_API
       // we are lucky
-      client->toWithTimeout(Serial, 500);
+      client->sendAll(Serial, 500);
 #else
       auto last = millis();
       while ((millis() - last) < 500) {

libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h

@@ -28,6 +28,7 @@
 #include "FS.h"
 #include "base64.h"
 #include "detail/RequestHandlersImpl.h"
+#include <StreamDev.h>
 
 static const char AUTHORIZATION_HEADER[] PROGMEM = "Authorization";
 static const char qop_auth[] PROGMEM = "qop=auth";
@@ -440,72 +441,69 @@ void ESP8266WebServerTemplate<ServerType>::_prepareHeader(String& response, int
     _responseHeaders = "";
 }
 
-template <typename ServerType>
-void ESP8266WebServerTemplate<ServerType>::send(int code, const char* content_type, const String& content) {
-    String header;
-    // Can we asume the following?
-    //if(code == 200 && content.length() == 0 && _contentLength == CONTENT_LENGTH_NOT_SET)
-    //  _contentLength = CONTENT_LENGTH_UNKNOWN;
-    _prepareHeader(header, code, content_type, content.length());
-    _currentClient.write((const uint8_t *)header.c_str(), header.length());
-    if(content.length())
-      sendContent(content);
-}
-
-template <typename ServerType>
-void ESP8266WebServerTemplate<ServerType>::send_P(int code, PGM_P content_type, PGM_P content) {
-    size_t contentLength = 0;
-
-    if (content != NULL) {
-        contentLength = strlen_P(content);
-    }
-
-    String header;
-    char type[64];
-    memccpy_P((void*)type, (PGM_VOID_P)content_type, 0, sizeof(type));
-    _prepareHeader(header, code, (const char* )type, contentLength);
-    _currentClient.write((const uint8_t *)header.c_str(), header.length());
-    if (contentLength) {
-        sendContent_P(content);
-    }
-}
-
-template <typename ServerType>
-void ESP8266WebServerTemplate<ServerType>::send_P(int code, PGM_P content_type, PGM_P content, size_t contentLength) {
-    String header;
-    char type[64];
-    memccpy_P((void*)type, (PGM_VOID_P)content_type, 0, sizeof(type));
-    _prepareHeader(header, code, (const char* )type, contentLength);
-    _currentClient.write((const uint8_t *)header.c_str(), header.length());
-    if (contentLength) {
-      sendContent_P(content, contentLength);
-    }
-}
-
 template <typename ServerType>
 void ESP8266WebServerTemplate<ServerType>::send(int code, char* content_type, const String& content) {
-  send(code, (const char*)content_type, content);
+  return send(code, (const char*)content_type, content);
+}
+
+template <typename ServerType>
+void ESP8266WebServerTemplate<ServerType>::send(int code, const char* content_type, const String& content) {
+  return send(code, content_type, content.c_str(), content.length());
 }
 
 template <typename ServerType>
 void ESP8266WebServerTemplate<ServerType>::send(int code, const String& content_type, const String& content) {
-  send(code, (const char*)content_type.c_str(), content);
+  return send(code, (const char*)content_type.c_str(), content);
 }
 
 template <typename ServerType>
 void ESP8266WebServerTemplate<ServerType>::sendContent(const String& content) {
-  if (_currentMethod == HTTP_HEAD) return;
+  StreamConstPtr ref(content.c_str(), content.length());
-  const char * footer = "\r\n";
+  sendContent(&ref);
-  size_t len = content.length();
+}
+
+template <typename ServerType>
+void ESP8266WebServerTemplate<ServerType>::send(int code, const char* content_type, Stream* stream, size_t content_length /*= 0*/) {
+  String header;
+  if (content_length == 0)
+      content_length = std::max((ssize_t)0, stream->streamRemaining());
+  _prepareHeader(header, code, content_type, content_length);
+  size_t sent = StreamConstPtr(header).sendAll(&_currentClient);
+  if (sent != header.length())
+      DBGWS("HTTPServer: error: sent %zd on %u bytes\n", sent, header.length());
+  if (content_length)
+    return sendContent(stream, content_length);
+}
+
+template <typename ServerType>
+void ESP8266WebServerTemplate<ServerType>::send_P(int code, PGM_P content_type, PGM_P content) {
+  StreamConstPtr ref(content, strlen_P(content));
+  return send(code, String(content_type).c_str(), &ref);
+}
+
+template <typename ServerType>
+void ESP8266WebServerTemplate<ServerType>::send_P(int code, PGM_P content_type, PGM_P content, size_t contentLength) {
+  StreamConstPtr ref(content, contentLength);
+  return send(code, String(content_type).c_str(), &ref);
+}
+
+template <typename ServerType>
+void ESP8266WebServerTemplate<ServerType>::sendContent(Stream* content, ssize_t content_length /* = 0*/) {
+  if (_currentMethod == HTTP_HEAD)
+    return;
+  if (content_length <= 0)
+    content_length = std::max((ssize_t)0, content->streamRemaining());
   if(_chunked) {
-    char chunkSize[11];
+    _currentClient.printf("%zx\r\n", content_length);
-    sprintf(chunkSize, "%zx\r\n", len);
-    _currentClient.write((const uint8_t *)chunkSize, strlen(chunkSize));
   }
-  _currentClient.write((const uint8_t *)content.c_str(), len);
+  ssize_t sent = content->sendSize(&_currentClient, content_length);
-  if(_chunked){
+  if (sent != content_length)
-    _currentClient.write((const uint8_t *)footer, 2);
+  {
-    if (len == 0) {
+    DBGWS("HTTPServer: error: short send after timeout (%d<%d)\n", sent, content_length);
+  }
+  if(_chunked) {
+    _currentClient.printf_P(PSTR("\r\n"));
+    if (content_length == 0) {
       _chunked = false;
     }
   }
@@ -518,19 +516,8 @@ void ESP8266WebServerTemplate<ServerType>::sendContent_P(PGM_P content) {
 
 template <typename ServerType>
 void ESP8266WebServerTemplate<ServerType>::sendContent_P(PGM_P content, size_t size) {
-  const char * footer = "\r\n";
+  StreamConstPtr ptr(content, size);
-  if(_chunked) {
+  return sendContent(&ptr, size);
-    char chunkSize[11];
-    sprintf(chunkSize, "%zx\r\n", size);
-    _currentClient.write((const uint8_t *)chunkSize, strlen(chunkSize));
-  }
-  _currentClient.write_P(content, size);
-  if(_chunked){
-    _currentClient.write((const uint8_t *)footer, 2);
-    if (size == 0) {
-      _chunked = false;
-    }
-  }
 }
 
 template <typename ServerType>
@@ -694,7 +681,7 @@ void ESP8266WebServerTemplate<ServerType>::_handleRequest() {
   }
   if (!handled) {
     using namespace mime;
-    send(404, String(FPSTR(mimeTable[html].mimeType)), String(F("Not found: ")) + _currentUri);
+    send(404, FPSTR(mimeTable[html].mimeType), String(F("Not found: ")) + _currentUri);
     handled = true;
   }
   if (handled) {

libraries/ESP8266WebServer/src/ESP8266WebServer.h

@@ -149,7 +149,7 @@ public:
   // code - HTTP response code, can be 200 or 404
   // content_type - HTTP content type, like "text/plain" or "image/png"
   // content - actual content body
-  void send(int code, const char* content_type = NULL, const String& content = String(""));
+  void send(int code, const char* content_type = NULL, const String& content = emptyString);
   void send(int code, char* content_type, const String& content);
   void send(int code, const String& content_type, const String& content);
   void send(int code, const char *content_type, const char *content) {
@@ -164,14 +164,23 @@ public:
   void send_P(int code, PGM_P content_type, PGM_P content);
   void send_P(int code, PGM_P content_type, PGM_P content, size_t contentLength);
 
+  void send(int code, const char* content_type, Stream* stream, size_t content_length = 0);
+  void send(int code, const char* content_type, Stream& stream, size_t content_length = 0);
+
   void setContentLength(const size_t contentLength);
   void sendHeader(const String& name, const String& value, bool first = false);
   void sendContent(const String& content);
+  void sendContent(String& content) {
+    sendContent((const String&)content);
+  }
   void sendContent_P(PGM_P content);
   void sendContent_P(PGM_P content, size_t size);
   void sendContent(const char *content) { sendContent_P(content); }
   void sendContent(const char *content, size_t size) { sendContent_P(content, size); }
 
+  void sendContent(Stream* content, ssize_t content_length = 0);
+  void sendContent(Stream& content, ssize_t content_length = 0) { sendContent(&content, content_length); }
+
   bool chunkedResponseModeStart_P (int code, PGM_P content_type) {
     if (_currentVersion == 0)
         // no chunk mode in HTTP/1.0
@@ -220,6 +229,30 @@ public:
     return contentLength;
   }
 
+  // Implement GET and HEAD requests for stream
+  // Stream body on HTTP_GET but not on HTTP_HEAD requests.
+  template<typename T>
+  size_t stream(T &aStream, const String& contentType, HTTPMethod requestMethod, ssize_t size) {
+    setContentLength(size);
+    send(200, contentType, emptyString);
+    if (requestMethod == HTTP_GET)
+        size = aStream.sendSize(_currentClient, size);
+    return size;
+  }
+
+  // Implement GET and HEAD requests for stream
+  // Stream body on HTTP_GET but not on HTTP_HEAD requests.
+  template<typename T>
+  size_t stream(T& aStream, const String& contentType, HTTPMethod requestMethod = HTTP_GET) {
+    ssize_t size = aStream.size();
+    if (size < 0)
+    {
+        send(500, F("text/html"), F("input stream: undetermined size"));
+        return 0;
+    }
+    return stream(aStream, contentType, requestMethod, size);
+  }
+
   static String responseCodeToString(const int code);
 
   void addHook (HookFunction hook) {

libraries/ESP8266WebServer/src/Parsing-impl.h

@@ -37,22 +37,8 @@ namespace esp8266webserver {
 template <typename ServerType>
 static bool readBytesWithTimeout(typename ServerType::ClientType& client, size_t maxLength, String& data, int timeout_ms)
 {
-  if (!data.reserve(maxLength + 1))
+  S2Stream dataStream(data);
-    return false;
+  return client.sendSize(dataStream, maxLength, timeout_ms) == maxLength;
-  data[0] = 0;  // data.clear()??
-  while (data.length() < maxLength) {
-    int tries = timeout_ms;
-    size_t avail;
-    while (!(avail = client.available()) && tries--)
-      delay(1);
-    if (!avail)
-      break;
-    if (data.length() + avail > maxLength)
-      avail = maxLength - data.length();
-    while (avail--)
-      data += (char)client.read();
-  }
-  return data.length() == maxLength;
 }
 
 template <typename ServerType>

libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino

@@ -0,0 +1,141 @@
+/*
+  WiFiEcho - Echo server
+
+  released to public domain
+*/
+
+#include <ESP8266WiFi.h>
+#include <ESP8266mDNS.h>
+#include <PolledTimeout.h>
+#include <algorithm> // std::min
+
+#ifndef STASSID
+#define STASSID "your-ssid"
+#define STAPSK  "your-password"
+#endif
+
+constexpr int port = 23;
+
+WiFiServer server(port);
+WiFiClient client;
+
+constexpr size_t sizes [] = { 0, 512, 384, 256, 128, 64, 16, 8, 4 };
+constexpr uint32_t breathMs = 200;
+esp8266::polledTimeout::oneShotFastMs enoughMs(breathMs);
+esp8266::polledTimeout::periodicFastMs test(2000);
+int t = 1; // test (1, 2 or 3, see below)
+int s = 0; // sizes[] index
+
+void setup() {
+
+  Serial.begin(115200);
+  Serial.println(ESP.getFullVersion());
+
+  WiFi.mode(WIFI_STA);
+  WiFi.begin(STASSID, STAPSK);
+  Serial.print("\nConnecting to ");
+  Serial.println(STASSID);
+  while (WiFi.status() != WL_CONNECTED) {
+    Serial.print('.');
+    delay(500);
+  }
+  Serial.println();
+  Serial.print("connected, address=");
+  Serial.println(WiFi.localIP());
+
+  server.begin();
+
+  MDNS.begin("echo23");
+
+  Serial.printf("Ready!\n"
+                "- Use 'telnet/nc echo23.local %d' to try echo\n\n"
+                "- Use 'python3 echo-client.py' bandwidth meter to compare transfer APIs\n\n"
+                "  and try typing 1, 1, 1, 2, 2, 2, 3, 3, 3 on console during transfers\n\n",
+                port);
+}
+
+
+void loop() {
+
+  MDNS.update();
+
+  static uint32_t tot = 0;
+  static uint32_t cnt = 0;
+  if (test && cnt) {
+    Serial.printf("measured-block-size=%u min-free-stack=%u", tot / cnt, ESP.getFreeContStack());
+    if (t == 2 && sizes[s]) {
+      Serial.printf(" (blocks: at most %d bytes)", sizes[s]);
+    }
+    if (t == 3 && sizes[s]) {
+      Serial.printf(" (blocks: exactly %d bytes)", sizes[s]);
+    }
+    if (t == 3 && !sizes[s]) {
+      Serial.printf(" (blocks: any size)");
+    }
+    Serial.printf("\n");
+  }
+
+  //check if there are any new clients
+  if (server.hasClient()) {
+    client = server.available();
+    Serial.println("New client");
+  }
+
+  if (Serial.available()) {
+    s = (s + 1) % (sizeof(sizes) / sizeof(sizes[0]));
+    switch (Serial.read()) {
+      case '1': if (t != 1) s = 0; t = 1; Serial.println("byte-by-byte (watch then press 2 or 3)"); break;
+      case '2': if (t != 2) s = 1; t = 2; Serial.printf("through buffer (watch then press 2 again, or 1 or 3)\n"); break;
+      case '3': if (t != 3) s = 0; t = 3; Serial.printf("direct access (watch then press 3 again, or 1 or 2)\n"); break;
+    }
+    tot = cnt = 0;
+    ESP.resetFreeContStack();
+  }
+
+  enoughMs.reset(breathMs);
+
+  if (t == 1) {
+    // byte by byte
+    while (client.available() && client.availableForWrite() && !enoughMs) {
+      // working char by char is not efficient
+      client.write(client.read());
+      cnt++;
+      tot += 1;
+    }
+  }
+
+  else if (t == 2) {
+    // block by block through a local buffer (2 copies)
+    while (client.available() && client.availableForWrite() && !enoughMs) {
+      size_t maxTo = std::min(client.available(), client.availableForWrite());
+      maxTo = std::min(maxTo, sizes[s]);
+      uint8_t buf[maxTo];
+      size_t tcp_got = client.read(buf, maxTo);
+      size_t tcp_sent = client.write(buf, tcp_got);
+      if (tcp_sent != maxTo) {
+        Serial.printf("len mismatch: available:%zd tcp-read:%zd serial-write:%zd\n", maxTo, tcp_got, tcp_sent);
+      }
+      tot += tcp_sent;
+      cnt++;
+    }
+  }
+
+  else if (t == 3) {
+    // stream to print, possibly with only one copy
+    if (sizes[s]) {
+      tot += client.sendSize(&client, sizes[s]);
+    } else {
+      tot += client.sendAll(&client);
+    }
+    cnt++;
+
+    switch (client.getLastSendReport()) {
+      case Stream::Report::Success: break;
+      case Stream::Report::TimedOut: Serial.println("Stream::send: timeout"); break;
+      case Stream::Report::ReadError: Serial.println("Stream::send: read error"); break;
+      case Stream::Report::WriteError: Serial.println("Stream::send: write error"); break;
+      case Stream::Report::ShortOperation: Serial.println("Stream::send: short transfer"); break;
+    }
+  }
+
+}

libraries/ESP8266WiFi/examples/WiFiEcho/echo-client.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python3
+
+import os
+import asyncio
+
+# 512 bytes
+message = bytearray(512);
+bufsize=len(message)
+print('message len=', bufsize)
+
+global recv
+recv = 0
+
+async def tcp_echo_open (ip, port):
+    return await asyncio.open_connection(ip, port)
+
+async def tcp_echo_sender(message, writer):
+    print('Writer started')
+    while True:
+        writer.write(message)
+        await writer.drain()
+
+async def tcp_echo_receiver(message, reader):
+    global recv
+    print('Reader started')
+    while True:
+        data = ''.encode('utf8')
+        while len(data) < bufsize:
+            data += await reader.read(bufsize - len(data))
+        recv += len(data);
+        if data != message:
+            print('error')
+
+async def tcp_stat():
+    global recv
+    dur = 0
+    loopsec = 2
+    while True:
+        last = recv
+        await asyncio.sleep(loopsec) # drifting
+        dur += loopsec
+        print('BW=', (recv - last) * 2 * 8 / 1024 / loopsec, 'Kibits/s avg=', recv * 2 * 8 / 1024 / dur)
+
+loop = asyncio.get_event_loop()
+reader, writer = loop.run_until_complete(tcp_echo_open('echo23.local', 23))
+loop.create_task(tcp_echo_receiver(message, reader))
+loop.create_task(tcp_echo_sender(message, writer))
+loop.create_task(tcp_stat())
+loop.run_forever()

libraries/ESP8266WiFi/src/CertStoreBearSSL.cpp

@@ -125,7 +125,7 @@ int CertStore::initCertStore(fs::FS &fs, const char *indexFileName, const char *
     uint8_t fileHeader[60];
     // 0..15 = filename in ASCII
     // 48...57 = length in decimal ASCII
-    uint32_t length;
+    int32_t length;
     if (data.read(fileHeader, sizeof(fileHeader)) != sizeof(fileHeader)) {
       break;
     }
@@ -201,7 +201,7 @@ const br_x509_trust_anchor *CertStore::findHashedTA(void *ctx, void *hashed_dn,
         free(der);
         return nullptr;
       }
-      if (data.read((uint8_t *)der, ci.length) != ci.length) {
+      if (data.read(der, ci.length) != (int)ci.length) {
         free(der);
         return nullptr;
       }

libraries/ESP8266WiFi/src/WiFiClient.cpp

@@ -244,7 +244,7 @@ size_t WiFiClient::write_P(PGM_P buf, size_t size)
 int WiFiClient::available()
 {
     if (!_client)
-        return false;
+        return 0;
 
     int result = _client->getSize();
 
@@ -262,10 +262,14 @@ int WiFiClient::read()
     return _client->read();
 }
 
-
 int WiFiClient::read(uint8_t* buf, size_t size)
 {
-    return (int) _client->read(reinterpret_cast<char*>(buf), size);
+    return (int)_client->read((char*)buf, size);
+}
+
+int WiFiClient::read(char* buf, size_t size)
+{
+    return (int)_client->read(buf, size);
 }
 
 int WiFiClient::peek()
@@ -412,3 +416,28 @@ uint8_t WiFiClient::getKeepAliveCount () const
 {
     return _client->getKeepAliveCount();
 }
+
+bool WiFiClient::hasPeekBufferAPI () const
+{
+    return true;
+}
+
+// return a pointer to available data buffer (size = peekAvailable())
+// semantic forbids any kind of read() before calling peekConsume()
+const char* WiFiClient::peekBuffer ()
+{
+    return _client? _client->peekBuffer(): nullptr;
+}
+
+// return number of byte accessible by peekBuffer()
+size_t WiFiClient::peekAvailable ()
+{
+    return _client? _client->peekAvailable(): 0;
+}
+
+// consume bytes after use (see peekBuffer)
+void WiFiClient::peekConsume (size_t consume)
+{
+    if (_client)
+        _client->peekConsume(consume);
+}

libraries/ESP8266WiFi/src/WiFiClient.h

@@ -66,7 +66,9 @@ public:
 
   virtual int available() override;
   virtual int read() override;
-  virtual int read(uint8_t *buf, size_t size) override;
+  virtual int read(uint8_t* buf, size_t size) override;
+  int read(char* buf, size_t size);
+
   virtual int peek() override;
   virtual size_t peekBytes(uint8_t *buffer, size_t length);
   size_t peekBytes(char *buffer, size_t length) {
@@ -120,6 +122,22 @@ public:
   bool getSync() const;
   void setSync(bool sync);
 
+  // peek buffer API is present
+  virtual bool hasPeekBufferAPI () const override;
+
+  // return number of byte accessible by peekBuffer()
+  virtual size_t peekAvailable () override;
+
+  // return a pointer to available data buffer (size = peekAvailable())
+  // semantic forbids any kind of read() before calling peekConsume()
+  virtual const char* peekBuffer () override;
+
+  // consume bytes after use (see peekBuffer)
+  virtual void peekConsume (size_t consume) override;
+
+  virtual bool outputCanTimeout () override { return connected(); }
+  virtual bool inputCanTimeout () override { return connected(); }
+
 protected:
 
   static int8_t _s_connected(void* arg, void* tpcb, int8_t err);

libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.cpp

@@ -362,6 +362,22 @@ int WiFiClientSecureCtx::read(uint8_t *buf, size_t size) {
   return 0; // If we're connected, no error but no read.
 }
 
+// return a pointer to available data buffer (size = peekAvailable())
+// semantic forbids any kind of read() before calling peekConsume()
+const char* WiFiClientSecureCtx::peekBuffer ()
+{
+    return (const char*)_recvapp_buf;
+}
+
+// consume bytes after use (see peekBuffer)
+void WiFiClientSecureCtx::peekConsume (size_t consume)
+{
+    // according to WiFiClientSecureCtx::read:
+    br_ssl_engine_recvapp_ack(_eng, consume);
+    _recvapp_buf = nullptr;
+    _recvapp_len = 0;
+}
+
 int WiFiClientSecureCtx::read() {
   uint8_t c;
   if (1 == read(&c, 1)) {

libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.h

@@ -48,6 +48,7 @@ class WiFiClientSecureCtx : public WiFiClient {
     size_t write_P(PGM_P buf, size_t size) override;
     size_t write(Stream& stream); // Note this is not virtual
     int read(uint8_t *buf, size_t size) override;
+    int read(char *buf, size_t size) { return read((uint8_t*)buf, size); }
     int available() override;
     int read() override;
     int peek() override;
@@ -120,6 +121,19 @@ class WiFiClientSecureCtx : public WiFiClient {
     bool setCiphers(const std::vector<uint16_t>& list);
     bool setCiphersLessSecure(); // Only use the limited set of RSA ciphers without EC
 
+    // peek buffer API is present
+    virtual bool hasPeekBufferAPI () const override { return true; }
+
+    // return number of byte accessible by peekBuffer()
+    virtual size_t peekAvailable () override { return WiFiClientSecureCtx::available(); }
+
+    // return a pointer to available data buffer (size = peekAvailable())
+    // semantic forbids any kind of read() before calling peekConsume()
+    virtual const char* peekBuffer () override;
+
+    // consume bytes after use (see peekBuffer)
+    virtual void peekConsume (size_t consume) override;
+
   protected:
     bool _connectSSL(const char *hostName); // Do initial SSL handshake
 
@@ -287,6 +301,19 @@ class WiFiClientSecure : public WiFiClient {
     static bool probeMaxFragmentLength(const char *hostname, uint16_t port, uint16_t len);
     static bool probeMaxFragmentLength(const String& host, uint16_t port, uint16_t len);
 
+    // peek buffer API is present
+    virtual bool hasPeekBufferAPI () const override { return true; }
+
+    // return number of byte accessible by peekBuffer()
+    virtual size_t peekAvailable () override { return _ctx->available(); }
+
+    // return a pointer to available data buffer (size = peekAvailable())
+    // semantic forbids any kind of read() before calling peekConsume()
+    virtual const char* peekBuffer () override { return _ctx->peekBuffer(); }
+
+    // consume bytes after use (see peekBuffer)
+    virtual void peekConsume (size_t consume) override { return _ctx->peekConsume(consume); }
+
   private:
     std::shared_ptr<WiFiClientSecureCtx> _ctx;
 

libraries/ESP8266WiFi/src/include/ClientContext.h

@@ -29,7 +29,8 @@ typedef void (*discard_cb_t)(void*, ClientContext*);
 extern "C" void esp_yield();
 extern "C" void esp_schedule();
 
-#include "DataSource.h"
+#include <assert.h>
+#include <StreamDev.h>
 
 bool getDefaultPrivateGlobalSyncValue ();
 
@@ -374,7 +375,8 @@ public:
         if (!_pcb) {
             return 0;
         }
-        return _write_from_source(new BufferDataSource(data, size));
+        StreamConstPtr ptr(data, size);
+        return _write_from_source(&ptr);
     }
 
     size_t write(Stream& stream)
@@ -382,7 +384,7 @@ public:
         if (!_pcb) {
             return 0;
         }
-        return _write_from_source(new BufferedStreamDataSource<Stream>(stream, stream.available()));
+        return _write_from_source(&stream);
     }
 
     size_t write_P(PGM_P buf, size_t size)
@@ -390,8 +392,8 @@ public:
         if (!_pcb) {
             return 0;
         }
-        ProgmemStream stream(buf, size);
+        StreamConstPtr ptr(buf, size);
-        return _write_from_source(new BufferedStreamDataSource<ProgmemStream>(stream, size));
+        return _write_from_source(&ptr);
     }
 
     void keepAlive (uint16_t idle_sec = TCP_DEFAULT_KEEPALIVE_IDLE_SEC, uint16_t intv_sec = TCP_DEFAULT_KEEPALIVE_INTERVAL_SEC, uint8_t count = TCP_DEFAULT_KEEPALIVE_COUNT)
@@ -436,6 +438,29 @@ public:
         _sync = sync;
     }
 
+    // return a pointer to available data buffer (size = peekAvailable())
+    // semantic forbids any kind of read() before calling peekConsume()
+    const char* peekBuffer ()
+    {
+        if (!_rx_buf)
+            return nullptr;
+        return (const char*)_rx_buf->payload + _rx_buf_offset;
+    }
+
+    // return number of byte accessible by peekBuffer()
+    size_t peekAvailable ()
+    {
+        if (!_rx_buf)
+            return 0;
+        return _rx_buf->len - _rx_buf_offset;
+    }
+
+    // consume bytes after use (see peekBuffer)
+    void peekConsume (size_t consume)
+    {
+        _consume(consume);
+    }
+
 protected:
 
     bool _is_timeout()
@@ -452,7 +477,7 @@ protected:
         }
     }
 
-    size_t _write_from_source(DataSource* ds)
+    size_t _write_from_source(Stream* ds)
     {
         assert(_datasource == nullptr);
         assert(!_send_waiting);
@@ -468,7 +493,6 @@ protected:
                 if (_is_timeout()) {
                     DEBUGV(":wtmo\r\n");
                 }
-                delete _datasource;
                 _datasource = nullptr;
                 break;
             }
@@ -495,20 +519,20 @@ protected:
             return false;
         }
 
-        DEBUGV(":wr %d %d\r\n", _datasource->available(), _written);
+        DEBUGV(":wr %d %d\r\n", _datasource->peekAvailable(), _written);
 
         bool has_written = false;
 
         while (_datasource) {
             if (state() == CLOSED)
                 return false;
-            size_t next_chunk_size = std::min((size_t)tcp_sndbuf(_pcb), _datasource->available());
+            size_t next_chunk_size = std::min((size_t)tcp_sndbuf(_pcb), _datasource->peekAvailable());
             if (!next_chunk_size)
                 break;
-            const uint8_t* buf = _datasource->get_buffer(next_chunk_size);
+            const char* buf = _datasource->peekBuffer();
 
             uint8_t flags = 0;
-            if (next_chunk_size < _datasource->available())
+            if (next_chunk_size < _datasource->peekAvailable())
                 //   PUSH is meant for peer, telling to give data to user app as soon as received
                 //   PUSH "may be set" when sender has finished sending a "meaningful" data block
                 //   PUSH does not break Nagle
@@ -522,15 +546,15 @@ protected:
 
             err_t err = tcp_write(_pcb, buf, next_chunk_size, flags);
 
-            DEBUGV(":wrc %d %d %d\r\n", next_chunk_size, _datasource->available(), (int)err);
+            DEBUGV(":wrc %d %d %d\r\n", next_chunk_size, _datasource->peekAvailable(), (int)err);
 
             if (err == ERR_OK) {
-                _datasource->release_buffer(buf, next_chunk_size);
+                _datasource->peekConsume(next_chunk_size);
                 _written += next_chunk_size;
                 has_written = true;
             } else {
-		// ERR_MEM(-1) is a valid error meaning
+                // ERR_MEM(-1) is a valid error meaning
-		// "come back later". It leaves state() opened
+                // "come back later". It leaves state() opened
                 break;
             }
         }
@@ -565,8 +589,6 @@ protected:
 
     void _consume(size_t size)
     {
-        if(_pcb)
-            tcp_recved(_pcb, size);
         ptrdiff_t left = _rx_buf->len - _rx_buf_offset - size;
         if(left > 0) {
             _rx_buf_offset += size;
@@ -583,6 +605,8 @@ protected:
             pbuf_ref(_rx_buf);
             pbuf_free(head);
         }
+        if(_pcb)
+            tcp_recved(_pcb, size);
     }
 
     err_t _recv(tcp_pcb* pcb, pbuf* pb, err_t err)
@@ -683,7 +707,7 @@ private:
     discard_cb_t _discard_cb;
     void* _discard_cb_arg;
 
-    DataSource* _datasource = nullptr;
+    Stream* _datasource = nullptr;
     size_t _written = 0;
     uint32_t _timeout_ms = 5000;
     uint32_t _op_start_time = 0;

libraries/ESP8266WiFi/src/include/DataSource.h

@@ -1,154 +0,0 @@
-/* DataSource.h - a read-only object similar to Stream, but with less methods
- * Copyright (c) 2016 Ivan Grokhotkov. All rights reserved.
- * This file is distributed under MIT license.
- */
-#ifndef DATASOURCE_H
-#define DATASOURCE_H
-
-#include <assert.h>
-
-class DataSource {
-public:
-    virtual ~DataSource() {}
-    virtual size_t available() = 0;
-    virtual const uint8_t* get_buffer(size_t size) = 0;
-    virtual void release_buffer(const uint8_t* buffer, size_t size) = 0;
-
-};
-
-class BufferDataSource : public DataSource {
-public:
-    BufferDataSource(const uint8_t* data, size_t size) :
-        _data(data),
-        _size(size)
-    {
-    }
-
-    size_t available() override
-    {
-        return _size - _pos;
-    }
-
-    const uint8_t* get_buffer(size_t size) override
-    {
-        (void)size;
-        assert(_pos + size <= _size);
-        return _data + _pos;
-    }
-
-    void release_buffer(const uint8_t* buffer, size_t size) override
-    {
-        (void)buffer;
-        assert(buffer == _data + _pos);
-        _pos += size;
-    }
-
-protected:
-    const uint8_t* _data;
-    const size_t _size;
-    size_t _pos = 0;
-};
-
-template<typename TStream>
-class BufferedStreamDataSource : public DataSource {
-public:
-    BufferedStreamDataSource(TStream& stream, size_t size) :
-        _stream(stream),
-        _size(size)
-    {
-    }
-
-    size_t available() override
-    {
-        return _size - _pos;
-    }
-
-    const uint8_t* get_buffer(size_t size) override
-    {
-        assert(_pos + size <= _size);
-
-        //Data that was already read from the stream but not released (e.g. if tcp_write error occured). Otherwise this should be 0.
-        const size_t stream_read = _streamPos - _pos;
-
-        //Min required buffer size: max(requested size, previous stream data already in buffer)
-        const size_t min_buffer_size = size > stream_read ? size : stream_read;
-
-        //Buffer too small?
-        if (_bufferSize < min_buffer_size) {
-            uint8_t *new_buffer = new uint8_t[min_buffer_size];
-            //If stream reading is ahead, than some data is already in the old buffer and needs to be copied to new resized buffer
-            if (_buffer && stream_read > 0) {
-                memcpy(new_buffer, _buffer.get(), stream_read);
-            }
-            _buffer.reset(new_buffer);
-            _bufferSize = min_buffer_size;
-        }
-
-        //Fetch remaining data from stream
-        //If error in tcp_write in ClientContext::_write_some() occured earlier and therefore release_buffer was not called last time, than the requested stream data is already in the buffer.
-        if (size > stream_read) {
-            //Remaining bytes to read from stream
-            const size_t stream_rem = size - stream_read;
-            const size_t cb = _stream.readBytes(reinterpret_cast<char*>(_buffer.get() + stream_read), stream_rem);
-            assert(cb == stream_rem);
-            (void)cb;
-            _streamPos += stream_rem;
-        }
-        return _buffer.get();
-
-    }
-
-    void release_buffer(const uint8_t* buffer, size_t size) override
-    {
-        if (size == 0) {
-            return;
-        }
-
-        (void)buffer;
-        _pos += size;     
-
-        //Cannot release more than acquired through get_buffer
-        assert(_pos <= _streamPos);
-
-        //Release less than requested with get_buffer?
-        if (_pos < _streamPos) {
-            // Move unreleased stream data in buffer to front
-            assert(_buffer);
-            memmove(_buffer.get(), _buffer.get() + size, _streamPos - _pos);
-        }
-    }
-
-protected:
-    TStream & _stream;
-    std::unique_ptr<uint8_t[]> _buffer;
-    size_t _size;
-    size_t _pos = 0;
-    size_t _bufferSize = 0;
-    size_t _streamPos = 0;
-};
-
-class ProgmemStream
-{
-public:
-    ProgmemStream(PGM_P buf, size_t size) :
-        _buf(buf),
-        _left(size)
-    {
-    }
-
-    size_t readBytes(char* dst, size_t size)
-    {
-        size_t will_read = (_left < size) ? _left : size;
-        memcpy_P((void*)dst, (PGM_VOID_P)_buf, will_read);
-        _left -= will_read;
-        _buf += will_read;
-        return will_read;
-    }
-
-protected:
-    PGM_P _buf;
-    size_t _left;
-};
-
-
-#endif //DATASOURCE_H

libraries/LittleFS/src/LittleFS.h

@@ -379,7 +379,7 @@ public:
         return result;
     }
 
-    size_t read(uint8_t* buf, size_t size) override {
+    int read(uint8_t* buf, size_t size) override {
         if (!_opened || !_fd | !buf) {
             return 0;
         }

libraries/Netdump/src/Netdump.h

@@ -75,7 +75,7 @@ private:
 
     WiFiClient tcpDumpClient;
     char* packetBuffer = nullptr;
-    size_t bufferIndex = 0;
+    int bufferIndex = 0;
 
     static constexpr int tcpBufferSize = 2048;
     static constexpr int maxPcapLength = 1024;

libraries/SDFS/src/SDFS.h

@@ -287,7 +287,7 @@ public:
         return _opened ? _fd->write(buf, size) : -1;
     }
 
-    size_t read(uint8_t* buf, size_t size) override
+    int read(uint8_t* buf, size_t size) override
     {
         return _opened ? _fd->read(buf, size) : -1;
     }

libraries/esp8266/examples/StreamString/StreamString.ino

@@ -0,0 +1,188 @@
+
+// this example sketch in the public domain is also a host and device test
+
+#include <StreamDev.h>
+#include <StreamString.h>
+
+void loop() {
+  delay(1000);
+}
+
+void checksketch(const char* what, const char* res1, const char* res2) {
+  if (strcmp(res1, res2) == 0) {
+    Serial << "PASSED: Test " << what << " (result: '" << res1 << "')\n";
+  } else {
+    Serial << "FAILED: Test " << what << ": '" << res1 << "' <> '" << res2 << "' !\n";
+  }
+}
+
+#ifndef check
+#define check(what, res1, res2) checksketch(what, res1, res2)
+#endif
+
+void testStringPtrProgmem() {
+  static const char inProgmem [] PROGMEM = "I am in progmem";
+  auto inProgmem2 = F("I am too in progmem");
+
+  int heap = (int)ESP.getFreeHeap();
+  auto stream1 = StreamConstPtr(inProgmem, sizeof(inProgmem) - 1);
+  auto stream2 = StreamConstPtr(inProgmem2);
+  Serial << stream1 << " - " << stream2 << "\n";
+  heap -= (int)ESP.getFreeHeap();
+  check("NO heap occupation while streaming progmem strings", String(heap).c_str(), "0");
+}
+
+void testStreamString() {
+  String inputString = "hello";
+  StreamString result;
+
+  // By default, reading a S2Stream(String) or a StreamString will consume the String.
+  // It can be disabled by calling ::resetPointer(), (not default)
+  // and reenabled by calling ::setConsume(). (default)
+  //
+  // In default consume mode, reading a byte or a block will remove it from
+  // the String.  Operations are O(n²).
+  //
+  // In non-default non-consume mode, it will just move a pointer.  That one
+  // can be ::resetPointer(pos) anytime.  See the example below.
+
+
+  // The String included in 'result' will not be modified by read:
+  // (this is not the default)
+  result.resetPointer();
+
+  {
+    // We use a a lighter StreamConstPtr(input) to make a read-only Stream out of
+    // a String that obviously should not be modified during the time the
+    // StreamConstPtr instance is used.  It is used as a source to be sent to
+    // 'result'.
+
+    result.clear();
+    StreamConstPtr(inputString).sendAll(result);
+    StreamConstPtr(inputString).sendAll(result);
+    StreamConstPtr(inputString).sendAll(result);
+    check("StreamConstPtr.sendAll(StreamString)", result.c_str(), "hellohellohello");
+  }
+
+  {
+    // equivalent of the above
+
+    result.clear();
+    result << inputString;
+    result << inputString << inputString;
+    check("StreamString<<String", result.c_str(), "hellohellohello");
+  }
+
+  {
+    // Now inputString is made into a Stream using S2Stream,
+    // and set in non-consume mode (using ::resetPointer()).
+
+    // Then, after that input is read once, it won't be anymore readable
+    // until the pointer is reset.
+
+    S2Stream input(inputString);
+    input.resetPointer();
+
+    result.clear();
+    input.sendAll(result);
+    input.sendAll(result);
+    check("S2Stream.sendAll(StreamString)", result.c_str(), "hello");
+    check("unmodified String given to S2Stream", inputString.c_str(), "hello");
+  }
+
+  {
+    // Same as above, with an offset
+
+    result.clear();
+    S2Stream input(inputString);
+    // stream position set to offset 2 (0 by default)
+    input.resetPointer(2);
+
+    input.sendAll(result);
+    input.sendAll(result);
+    check("S2Stream.resetPointer(2):", result.c_str(), "llo");
+  }
+
+  {
+    // inputString made into a Stream
+    // reading the Stream consumes the String
+
+    result.clear();
+    S2Stream input(inputString);
+    // reading stream will consume the string
+    input.setConsume(); // can be ommitted, this is the default
+
+    input.sendSize(result, 1);
+    input.sendSize(result, 2);
+    check("setConsume(): S2Stream().sendSize(StreamString,3)", result.c_str(), "hel");
+    check("setConsume(): String given from S2Stream is swallowed", inputString.c_str(), "lo");
+  }
+
+  // Streaming with common String constructors
+  {
+    StreamString cons(inputString);
+    check("StreamString(String)", cons.c_str(), inputString.c_str());
+  }
+  {
+    StreamString cons(result);
+    check("StreamString(char*)", cons.c_str(), result.c_str());
+  }
+  {
+    StreamString cons("abc");
+    check("StreamString(char*)", cons.c_str(), "abc");
+  }
+  {
+    StreamString cons(F("abc"));
+    check("StreamString(F())", cons.c_str(), "abc");
+  }
+  {
+    StreamString cons(23);
+    check("StreamString(int)", cons.c_str(), "23");
+  }
+  {
+    StreamString cons('a');
+    check("StreamString(char)", cons.c_str(), "a");
+  }
+  {
+    StreamString cons(23.2);
+    check("StreamString(float)", cons.c_str(), "23.20");
+  }
+
+#if !CORE_MOCK
+
+  // A progmem won't use Heap when StringPtr is used
+  testStringPtrProgmem();
+
+  // .. but it does when S2Stream or StreamString is used
+  {
+    int heap = (int)ESP.getFreeHeap();
+    auto stream = StreamString(F("I am in progmem"));
+    Serial << stream << "\n";
+    heap -= (int)ESP.getFreeHeap();
+    String heapStr(heap);
+    if (heap != 0) {
+      check("heap is occupied by String/StreamString(progmem)", heapStr.c_str(), heapStr.c_str());
+    } else {
+      check("ERROR: heap should be occupied by String/StreamString(progmem)", heapStr.c_str(), "-1");
+    }
+  }
+
+  // (check again to be sure)
+  testStringPtrProgmem();
+
+#endif
+}
+
+#ifndef TEST_CASE
+
+void setup() {
+  Serial.begin(115200);
+  delay(1000);
+
+  testStreamString();
+
+  Serial.printf("sizeof: String:%d Stream:%d StreamString:%d SStream:%d\n",
+                (int)sizeof(String), (int)sizeof(Stream), (int)sizeof(StreamString), (int)sizeof(S2Stream));
+}
+
+#endif

tests/device/Makefile

@@ -64,7 +64,7 @@ $(TEST_LIST):
 	$(SILENT)mkdir -p $(LOCAL_BUILD_DIR)
 ifeq ("$(MOCK)", "1")
 	@echo Compiling $(notdir $@)
-	(cd ../host; make ULIBDIRS=../device/libraries/BSTest ../device/$(@:%.ino=%))
+	(cd ../host; make D=$(V) ULIBDIRS=../device/libraries/BSTest ../device/$(@:%.ino=%))
 	$(SILENT)source $(BS_DIR)/virtualenv/bin/activate && \
 		$(PYTHON) $(BS_DIR)/runner.py \
 			$(RUNNER_DEBUG_FLAG) \

tests/device/test_sw_StreamString/test_sw_StreamString.ino

@@ -0,0 +1,25 @@
+
+#include <Arduino.h>
+#include <BSTest.h>
+
+#define check(what, res1, res2) CHECK(strcmp(res1, res2) == 0)
+
+#include "../../../libraries/esp8266/examples/StreamString/StreamString.ino"
+
+BS_ENV_DECLARE();
+
+bool pretest ()
+{
+    return true;
+}
+
+void setup ()
+{
+    Serial.begin(115200);
+    BS_RUN(Serial);
+}
+
+TEST_CASE("StreamString tests", "[StreamString]")
+{
+    testStream();
+}

tests/device/test_sw_WiFiServer/test_sw_WiFiServer.py

@@ -11,6 +11,7 @@ def setup_echo_server(e):
     global stop_client_thread
     global client_thread
     def echo_client_thread():
+        time.sleep(1) # let some time for mDNS to start
         server_address = socket.gethostbyname('esp8266-wfs-test.local')
         count = 0
         while count < 5 and not stop_client_thread:

tests/host/Makefile

@@ -76,7 +76,7 @@ $(shell mkdir -p $(BINDIR))
 CORE_CPP_FILES := \
 	$(addprefix $(abspath $(CORE_PATH))/,\
 		debug.cpp \
-		StreamString.cpp \
+		StreamSend.cpp \
 		Stream.cpp \
 		WString.cpp \
 		Print.cpp \

tests/host/common/Arduino.cpp

@@ -80,4 +80,3 @@ cont_t* g_pcont = NULL;
 extern "C" void cont_yield(cont_t*)
 {
 }
-

tests/host/common/ClientContextSocket.cpp

@@ -89,7 +89,8 @@ ssize_t mockFillInBuf (int sock, char* ccinbuf, size_t& ccinbufsize)
 	if (ret == 0)
 	{
 		// connection closed
-		return -1;
+		// nothing is read
+		return 0;
 	}
 
 	if (ret == -1)
@@ -97,16 +98,20 @@ ssize_t mockFillInBuf (int sock, char* ccinbuf, size_t& ccinbufsize)
 		if (errno != EAGAIN)
 		{
 			fprintf(stderr, MOCK "ClientContext::(read/peek fd=%i): filling buffer for %zd bytes: %s\n", sock, maxread, strerror(errno));
+            // error
 			return -1;
 		}
 		ret = 0;
 	}
+
 	ccinbufsize += ret;
 	return ret;
 }
 
 ssize_t mockPeekBytes (int sock, char* dst, size_t usersize, int timeout_ms, char* ccinbuf, size_t& ccinbufsize)
 {
+    // usersize==0: peekAvailable()
+
 	if (usersize > CCBUFSIZE)
 		mockverbose("CCBUFSIZE(%d) should be increased by %zd bytes (-> %zd)\n", CCBUFSIZE, usersize - CCBUFSIZE, usersize);
 
@@ -114,7 +119,7 @@ ssize_t mockPeekBytes (int sock, char* dst, size_t usersize, int timeout_ms, cha
 	size_t retsize = 0;
 	do
 	{
-		if (usersize <= ccinbufsize)
+		if (usersize && usersize <= ccinbufsize)
 		{
 			// data already buffered
 			retsize = usersize;
@@ -123,7 +128,14 @@ ssize_t mockPeekBytes (int sock, char* dst, size_t usersize, int timeout_ms, cha
 		
 		// check incoming data data
 		if (mockFillInBuf(sock, ccinbuf, ccinbufsize) < 0)
+		{
 			return -1;
+	    }
+
+        if (usersize == 0 && ccinbufsize)
+            // peekAvailable
+            return ccinbufsize;
+
 		if (usersize <= ccinbufsize)
 		{
 			// data just received
@@ -179,7 +191,7 @@ ssize_t mockWrite (int sock, const uint8_t* data, size_t size, int timeout_ms)
 #endif
 			if (ret == -1)
 			{
-				fprintf(stderr, MOCK "ClientContext::read: write(%d): %s\n", sock, strerror(errno));
+				fprintf(stderr, MOCK "ClientContext::write/send(%d): %s\n", sock, strerror(errno));
 				return -1;
 			}
 			sent += ret;
@@ -187,6 +199,8 @@ ssize_t mockWrite (int sock, const uint8_t* data, size_t size, int timeout_ms)
 				fprintf(stderr, MOCK "ClientContext::write: sent %d bytes (%zd / %zd)\n", ret, sent, size);
 		}
 	}
-	fprintf(stderr, MOCK "ClientContext::write: total sent %zd bytes\n", sent);
+#ifdef DEBUG_ESP_WIFI
+    mockverbose(MOCK "ClientContext::write: total sent %zd bytes\n", sent);
+#endif
 	return sent;
 }

tests/host/common/MockUART.cpp

@@ -491,3 +491,9 @@ uart_detect_baudrate(int uart_nr)
 }
 
 };
+
+
+size_t uart_peek_available (uart_t* uart) { return 0; }
+const char* uart_peek_buffer (uart_t* uart) { return nullptr; }
+void uart_peek_consume (uart_t* uart, size_t consume) { (void)uart; (void)consume; }
+

tests/host/common/include/ClientContext.h

@@ -27,7 +27,7 @@ class WiFiClient;
 extern "C" void esp_yield();
 extern "C" void esp_schedule();
 
-#include <include/DataSource.h>
+#include <assert.h>
 
 bool getDefaultPrivateGlobalSyncValue ();
 
@@ -300,6 +300,33 @@ public:
         _sync = sync;
     }
 
+    // return a pointer to available data buffer (size = peekAvailable())
+    // semantic forbids any kind of read() before calling peekConsume()
+    const char* peekBuffer ()
+    {
+        return _inbuf;
+    }
+
+    // return number of byte accessible by peekBuffer()
+    size_t peekAvailable ()
+    {
+        ssize_t ret = mockPeekBytes(_sock, nullptr, 0, 0, _inbuf, _inbufsize);
+        if (ret < 0)
+        {
+            abort();
+            return 0;
+        }
+        return _inbufsize;
+    }
+
+    // consume bytes after use (see peekBuffer)
+    void peekConsume (size_t consume)
+    {
+        assert(consume <= _inbufsize);
+        memmove(_inbuf, _inbuf + consume, _inbufsize - consume);
+        _inbufsize -= consume;
+    }
+
 private:
 
     discard_cb_t _discard_cb = nullptr;

tests/restyle.sh

@@ -15,8 +15,10 @@ libraries/ESP8266mDNS
 libraries/Wire
 libraries/lwIP*
 cores/esp8266/Lwip*
-cores/esp8266/core_esp8266_si2c.cpp
 cores/esp8266/debug*
+cores/esp8266/core_esp8266_si2c.cpp
+cores/esp8266/StreamString.*
+cores/esp8266/StreamSend.*
 libraries/Netdump
 "