uart: BW improvements (#4620)

* uart fixes and BW improvements

* uart: read_char straightly use hw buffer

* +attributes for functions called by ISR

* uart: BW improvements
read_char straightly use hw buffer (+ ~10%bw)
read by block (+ ~190%bw) (instead of generic Stream::readBytes)
attributes for functions called by ISR
remove overrun message
remove some ISR flags which were not honoured

* fix merge

* fix buffer overflow

* serial stress test sketch

* astyle

* serial stress example: interactive keyboard, stop reading, overrun

* serial device test: bandwidth & overrun

* update + HardwareSerial::hasError()

* interactive overrun in example

* astyle

* Test using @plerup's SoftwareSerial as submodule (tag 3.4.1)

* update upstream ref (fix warning)

* host mock uart/read(buf,size)

* reset style changes in submodules before style diff

* update build_boards_manager_package.sh for submodules

* trigger CI (removing space)

* cannot reproduce locally the CI issue, setting bash -x option to get live trace

* remove previously added (in this PR) 'set -e' in package builder (passes local tests, not real CI)
script-comment new recipe.hooks.core.prebuild.3 (along with already commented .1 and .2)
moved CI package test to be first on the test list
remove 'set -x', wish me luck

.gitmodules

@@ -4,3 +4,6 @@
 [submodule "tools/sdk/ssl/bearssl"]
 	path = tools/sdk/ssl/bearssl
 	url = https://github.com/earlephilhower/bearssl-esp8266
+[submodule "libraries/SoftwareSerial"]
+	path = libraries/SoftwareSerial
+	url = https://github.com/plerup/espsoftwareserial.git

.travis.yml

@@ -8,6 +8,8 @@ cache:
 
 matrix:
   include:
+    - env:
+      - BUILD_TYPE=package
     - env:
       - BUILD_TYPE=build_even
     - env:
@@ -22,8 +24,6 @@ matrix:
       - BUILD_TYPE=platformio_odd
     - env:
       - BUILD_TYPE=docs
-    - env:
-      - BUILD_TYPE=package
     - env:
       - BUILD_TYPE=host_tests
     - env:

cores/esp8266/HardwareSerial.h

@@ -88,6 +88,10 @@ public:
     void end();
 
     size_t setRxBufferSize(size_t size);
+    size_t getRxBufferSize()
+    {
+        return uart_get_rx_buffer_size(_uart);
+    }
 
     void swap()
     {
@@ -120,14 +124,22 @@ public:
 
     int peek(void) override
     {
-        // this may return -1, but that's okay
+        // return -1 when data is unvailable (arduino api)
         return uart_peek_char(_uart);
     }
     int read(void) override
     {
-        // this may return -1, but that's okay
+        // return -1 when data is unvailable (arduino api)
         return uart_read_char(_uart);
     }
+    size_t readBytes(char* buffer, size_t size) override
+    {
+        return uart_read(_uart, buffer, size);
+    }
+    size_t readBytes(uint8_t* buffer, size_t size) override
+    {
+        return uart_read(_uart, (char*)buffer, size);
+    }
     int availableForWrite(void)
     {
         return static_cast<int>(uart_tx_free(_uart));
@@ -184,6 +196,11 @@ public:
         return uart_has_overrun(_uart);
     }
 
+    bool hasRxError(void)
+    {
+        return uart_has_rx_error(_uart);
+    }
+
     void startDetectBaudrate();
 
     unsigned long testBaudrate();

cores/esp8266/uart.c

@@ -47,11 +47,11 @@
 #include "user_interface.h"
 #include "uart_register.h"
 
-const char overrun_str [] PROGMEM STORE_ATTR = "uart input full!\r\n";
+//const char overrun_str [] PROGMEM STORE_ATTR = "uart input full!\r\n";
 static int s_uart_debug_nr = UART0;
 
 
-struct uart_rx_buffer_ 
+struct uart_rx_buffer_
 {
     size_t size;
     size_t rpos;
@@ -65,7 +65,8 @@ struct uart_
     int baud_rate;
     bool rx_enabled;
     bool tx_enabled;
-    bool overrun;
+    bool rx_overrun;
+    bool rx_error;
     uint8_t rx_pin;
     uint8_t tx_pin;
     struct uart_rx_buffer_ * rx_buffer;
@@ -85,7 +86,8 @@ struct uart_
 
 
 
-inline size_t 
+// called by ISR
+inline size_t ICACHE_RAM_ATTR
 uart_rx_fifo_available(const int uart_nr) 
 {
     return (USS(uart_nr) >> USRXC) & 0xFF;
@@ -110,11 +112,11 @@ uart_rx_available_unsafe(uart_t* uart)
     return uart_rx_buffer_available_unsafe(uart->rx_buffer) + uart_rx_fifo_available(uart->uart_nr);
 }
 
-
 //#define UART_DISCARD_NEWEST
 
 // Copy all the rx fifo bytes that fit into the rx buffer
-inline void 
+// called by ISR
+inline void ICACHE_RAM_ATTR
 uart_rx_copy_fifo_to_buffer_unsafe(uart_t* uart) 
 {
     struct uart_rx_buffer_ *rx_buffer = uart->rx_buffer;
@@ -124,11 +126,10 @@ uart_rx_copy_fifo_to_buffer_unsafe(uart_t* uart)
         size_t nextPos = (rx_buffer->wpos + 1) % rx_buffer->size;
         if(nextPos == rx_buffer->rpos) 
         {
-
-            if (!uart->overrun) 
+            if (!uart->rx_overrun)
             {
-                uart->overrun = true;
-                os_printf_plus(overrun_str);
+                uart->rx_overrun = true;
+                //os_printf_plus(overrun_str);
             }
 
             // a choice has to be made here,
@@ -158,25 +159,27 @@ uart_peek_char_unsafe(uart_t* uart)
    
     //without the following if statement and body, there is a good chance of a fifo overrun
     if (uart_rx_buffer_available_unsafe(uart->rx_buffer) == 0)
+        // hw fifo can't be peeked, data need to be copied to sw
         uart_rx_copy_fifo_to_buffer_unsafe(uart);
     
     return uart->rx_buffer->buffer[uart->rx_buffer->rpos];
 }
 
-inline int 
+// taking data straight from hw fifo: loopback-test BW jumps by 19%
+inline int
 uart_read_char_unsafe(uart_t* uart)
 {
-    int data = uart_peek_char_unsafe(uart);
-    if(data != -1)
+    if (uart_rx_buffer_available_unsafe(uart->rx_buffer))
+    {
+        // take oldest sw data
+        int ret = uart->rx_buffer->buffer[uart->rx_buffer->rpos];
         uart->rx_buffer->rpos = (uart->rx_buffer->rpos + 1) % uart->rx_buffer->size;
-    return data;
+        return ret;
+    }
+    // unavailable
+    return -1;
 }
 
-
-/**********************************************************/
-
-
-
 size_t 
 uart_rx_available(uart_t* uart)
 {
@@ -204,14 +207,47 @@ uart_peek_char(uart_t* uart)
 
 int 
 uart_read_char(uart_t* uart)
+{
+    uint8_t ret;
+    return uart_read(uart, (char*)&ret, 1)? ret: -1;
+}
+
+// loopback-test BW jumps by 190%
+size_t
+uart_read(uart_t* uart, char* userbuffer, size_t usersize)
 {
     if(uart == NULL || !uart->rx_enabled)
-        return -1;
-    
+        return 0;
+
+    size_t ret = 0;
     ETS_UART_INTR_DISABLE();
-    int data = uart_read_char_unsafe(uart);
+
+    while (ret < usersize && uart_rx_available_unsafe(uart))
+    {
+        if (!uart_rx_buffer_available_unsafe(uart->rx_buffer))
+        {
+            // no more data in sw buffer, take them from hw fifo
+            while (ret < usersize && uart_rx_fifo_available(uart->uart_nr))
+                userbuffer[ret++] = USF(uart->uart_nr);
+
+	    // no more sw/hw data available
+            break;
+        }
+
+        // pour sw buffer to user's buffer
+        // get largest linear length from sw buffer
+        size_t chunk = uart->rx_buffer->rpos < uart->rx_buffer->wpos?
+                           uart->rx_buffer->wpos - uart->rx_buffer->rpos:
+                           uart->rx_buffer->size - uart->rx_buffer->rpos;
+        if (ret + chunk > usersize)
+            chunk = usersize - ret;
+        memcpy(userbuffer + ret, uart->rx_buffer->buffer + uart->rx_buffer->rpos, chunk);
+        uart->rx_buffer->rpos = (uart->rx_buffer->rpos + chunk) % uart->rx_buffer->size;
+        ret += chunk;
+    }
+
     ETS_UART_INTR_ENABLE();
-    return data;
+    return ret;
 }
 
 size_t 
@@ -231,6 +267,8 @@ uart_resize_rx_buffer(uart_t* uart, size_t new_size)
     ETS_UART_INTR_DISABLE();
     while(uart_rx_available_unsafe(uart) && new_wpos < new_size)
         new_buf[new_wpos++] = uart_read_char_unsafe(uart); //if uart_rx_available_unsafe() returns non-0, uart_read_char_unsafe() can't return -1
+    if (new_wpos == new_size)
+        new_wpos = 0;
     
     uint8_t * old_buf = uart->rx_buffer->buffer;
     uart->rx_buffer->rpos = 0;
@@ -242,22 +280,39 @@ uart_resize_rx_buffer(uart_t* uart, size_t new_size)
     return uart->rx_buffer->size;
 }
 
+size_t
+uart_get_rx_buffer_size(uart_t* uart)
+{
+    return uart && uart->rx_enabled? uart->rx_buffer->size: 0;
+}
 
 
 void ICACHE_RAM_ATTR 
 uart_isr(void * arg)
 {
     uart_t* uart = (uart_t*)arg;
+    uint32_t usis = USIS(uart->uart_nr);
+
     if(uart == NULL || !uart->rx_enabled) 
     {
-        USIC(uart->uart_nr) = USIS(uart->uart_nr);
+        USIC(uart->uart_nr) = usis;
         ETS_UART_INTR_DISABLE();
         return;
     }
-    if(USIS(uart->uart_nr) & ((1 << UIFF) | (1 << UITO)))
+
+    if(usis & (1 << UIFF))
         uart_rx_copy_fifo_to_buffer_unsafe(uart);
+
+    if((usis & (1 << UIOF)) && !uart->rx_overrun)
+    {
+        uart->rx_overrun = true;
+        //os_printf_plus(overrun_str);
+    }
     
-    USIC(uart->uart_nr) = USIS(uart->uart_nr);
+    if (usis & ((1 << UIFR) | (1 << UIPE) | (1 << UITO)))
+        uart->rx_error = true;
+
+    USIC(uart->uart_nr) = usis;
 }
 
 static void 
@@ -270,9 +325,22 @@ uart_start_isr(uart_t* uart)
     // triggers the IRS very often.  A value of 127 would not leave much time
     // for ISR to clear fifo before the next byte is dropped.  So pick a value
     // in the middle.
-    USC1(uart->uart_nr) = (100   << UCFFT) | (0x02 << UCTOT) | (1 <<UCTOE );
+    // update: loopback test @ 3Mbauds/8n1 (=2343Kibits/s):
+    // - 4..120 give > 2300Kibits/s
+    // - 1, 2, 3 are below
+    // was 100, use 16 to stay away from overrun
+    #define INTRIGG 16
+
+    //was:USC1(uart->uart_nr) = (INTRIGG << UCFFT) | (0x02 << UCTOT) | (1 <<UCTOE);
+    USC1(uart->uart_nr) = (INTRIGG << UCFFT);
     USIC(uart->uart_nr) = 0xffff;
-    USIE(uart->uart_nr) = (1 << UIFF) | (1 << UIFR) | (1 << UITO);
+    //was: USIE(uart->uart_nr) = (1 << UIFF) | (1 << UIFR) | (1 << UITO);
+    // UIFF: rx fifo full
+    // UIOF: rx fifo overflow (=overrun)
+    // UIFR: frame error
+    // UIPE: parity error
+    // UITO: rx fifo timeout
+    USIE(uart->uart_nr) = (1 << UIFF) | (1 << UIOF) | (1 << UIFR) | (1 << UIPE) | (1 << UITO);
     ETS_UART_INTR_ATTACH(uart_isr,  (void *)uart);
     ETS_UART_INTR_ENABLE();
 }
@@ -415,7 +483,8 @@ uart_init(int uart_nr, int baudrate, int config, int mode, int tx_pin, size_t rx
         return NULL;
 
     uart->uart_nr = uart_nr;
-    uart->overrun = false;
+    uart->rx_overrun = false;
+    uart->rx_error = false;
 
     switch(uart->uart_nr) 
     {
@@ -678,11 +747,22 @@ uart_rx_enabled(uart_t* uart)
 bool 
 uart_has_overrun (uart_t* uart)
 {
-    if (uart == NULL || !uart->overrun)
+    if (uart == NULL || !uart->rx_overrun)
         return false;
 
     // clear flag
-    uart->overrun = false;
+    uart->rx_overrun = false;
+    return true;
+}
+
+bool
+uart_has_rx_error (uart_t* uart)
+{
+    if (uart == NULL || !uart->rx_error)
+        return false;
+
+    // clear flag
+    uart->rx_error = false;
     return true;
 }
 

cores/esp8266/uart.h

@@ -126,17 +126,20 @@ void uart_set_baudrate(uart_t* uart, int baud_rate);
 int uart_get_baudrate(uart_t* uart);
 
 size_t uart_resize_rx_buffer(uart_t* uart, size_t new_size);
+size_t uart_get_rx_buffer_size(uart_t* uart);
 
 size_t uart_write_char(uart_t* uart, char c);
 size_t uart_write(uart_t* uart, const char* buf, size_t size);
 int uart_read_char(uart_t* uart);
 int uart_peek_char(uart_t* uart);
+size_t uart_read(uart_t* uart, char* buffer, size_t size);
 size_t uart_rx_available(uart_t* uart);
 size_t uart_tx_free(uart_t* uart);
 void uart_wait_tx_empty(uart_t* uart);
 void uart_flush(uart_t* uart);
 
 bool uart_has_overrun (uart_t* uart); // returns then clear overrun flag
+bool uart_has_rx_error (uart_t* uart); // returns then clear rxerror flag
 
 void uart_set_debug(int uart_nr);
 int uart_get_debug();

libraries/SoftwareSerial

@@ -0,0 +1 @@
+Subproject commit 23ae000cb2cf4d5823a2744f6b8ae831575ff135

libraries/esp8266/examples/SerialStress/SerialStress.ino

@@ -0,0 +1,187 @@
+
+/*
+  Serial read/write/verify/benchmark
+  Using internal loopback
+  Using SoftwareSerial library for logging
+
+  Sketch meant for debugging only
+  Released to public domain
+*/
+
+#include <ESP8266WiFi.h>
+#include <SoftwareSerial.h>
+
+#define SSBAUD          115200  // logger on console for humans
+#define BAUD            3000000 // hardware serial stress test
+#define BUFFER_SIZE     4096    // may be useless to use more than 2*SERIAL_SIZE_RX
+#define SERIAL_SIZE_RX  1024    // Serial.setRxBufferSize()
+
+#define TIMEOUT 5000
+#define DEBUG(x...) //x
+
+uint8_t buf [BUFFER_SIZE];
+uint8_t temp [BUFFER_SIZE];
+bool reading = true;
+
+static size_t out_idx = 0, in_idx = 0;
+static size_t local_receive_size = 0;
+static size_t size_for_1sec, size_for_led = 0;
+static size_t maxavail = 0;
+static uint64_t in_total = 0, in_prev = 0;
+static uint64_t start_ms, last_ms;
+static uint64_t timeout;
+
+Stream* logger;
+
+void error(const char* what) {
+  logger->printf("\nerror: %s after %ld minutes\nread idx:  %d\nwrite idx: %d\ntotal:     %ld\nlast read: %d\nmaxavail:  %d\n",
+                 what, (long)((millis() - start_ms) / 60000), in_idx, out_idx, (long)in_total, (int)local_receive_size, maxavail);
+  if (Serial.hasOverrun()) {
+    logger->printf("overrun!\n");
+  }
+  logger->printf("should be (size=%d idx=%d..%d):\n    ", BUFFER_SIZE, in_idx, in_idx + local_receive_size - 1);
+  for (size_t i = in_idx; i < in_idx + local_receive_size; i++) {
+    logger->printf("%02x(%c) ", buf[i], (unsigned char)((buf[i] > 31 && buf[i] < 128) ? buf[i] : '.'));
+  }
+  logger->print("\n\nis: ");
+  for (size_t i = 0; i < local_receive_size; i++) {
+    logger->printf("%02x(%c) ", temp[i], (unsigned char)((temp[i] > 31 && temp[i] < 128) ? temp[i] : '.'));
+  }
+  logger->println("\n\n");
+
+  while (true) {
+    delay(1000);
+  }
+}
+
+void preinit() {
+  // (no C++ in function)
+  // disable wifi
+  ESP8266WiFiClass::preinitWiFiOff();
+}
+
+void setup() {
+  pinMode(LED_BUILTIN, OUTPUT);
+
+  Serial.begin(BAUD);
+  Serial.swap(); // RX=GPIO13 TX=GPIO15
+  Serial.setRxBufferSize(SERIAL_SIZE_RX);
+
+  // using HardwareSerial0 pins,
+  // so we can still log to the regular usbserial chips
+  SoftwareSerial* ss = new SoftwareSerial(3, 1);
+  ss->begin(SSBAUD);
+  logger = ss;
+  logger->println();
+  logger->printf("\n\nOn Software Serial for logging\n");
+
+  int baud = Serial.baudRate();
+  logger->printf(ESP.getFullVersion().c_str());
+  logger->printf("\n\nBAUD: %d - CoreRxBuffer: %d bytes - TestBuffer: %d bytes\n",
+                 baud, SERIAL_SIZE_RX, BUFFER_SIZE);
+
+  size_for_1sec = baud / 10; // 8n1=10baudFor8bits
+  logger->printf("led changes state every %zd bytes (= 1 second)\n", size_for_1sec);
+  logger->printf("press 's' to stop reading, not writing (induces overrun)\n");
+
+  // prepare send/compare buffer
+  for (size_t i = 0; i < sizeof buf; i++) {
+    buf[i] = (uint8_t)i;
+  }
+
+  // bind RX and TX
+  USC0(0) |= (1 << UCLBE);
+
+  while (Serial.read() == -1);
+  if (Serial.hasOverrun()) {
+    logger->print("overrun?\n");
+  }
+
+  timeout = (start_ms = last_ms = millis()) + TIMEOUT;
+  logger->println("setup done");
+}
+
+void loop() {
+  size_t maxlen = Serial.availableForWrite();
+  // check remaining space in buffer
+  if (maxlen > BUFFER_SIZE - out_idx) {
+    maxlen = BUFFER_SIZE - out_idx;
+  }
+  // check if not cycling more than buffer size relatively to input
+  size_t in_out = out_idx == in_idx ?
+                  BUFFER_SIZE :
+                  (in_idx + BUFFER_SIZE - out_idx - 1) % BUFFER_SIZE;
+  if (maxlen > in_out) {
+    maxlen = in_out;
+  }
+  DEBUG(logger->printf("(aw%i/w%i", Serial.availableForWrite(), maxlen));
+  size_t local_written_size = Serial.write(buf + out_idx, maxlen);
+  DEBUG(logger->printf(":w%i/aw%i/ar%i)\n", local_written_size, Serial.availableForWrite(), Serial.available()));
+  if (local_written_size > maxlen) {
+    error("bad write");
+  }
+  if ((out_idx += local_written_size) == BUFFER_SIZE) {
+    out_idx = 0;
+  }
+  delay(0);
+
+  DEBUG(logger->printf("----------\n"));
+
+  if (Serial.hasOverrun()) {
+    logger->printf("rx overrun!\n");
+  }
+  if (Serial.hasRxError()) {
+    logger->printf("rx error!\n");
+  }
+
+  if (reading) {
+    // receive data
+    maxlen = Serial.available();
+    if (maxlen > maxavail) {
+      maxavail = maxlen;
+    }
+    // check space in temp receive buffer
+    if (maxlen > BUFFER_SIZE - in_idx) {
+      maxlen = BUFFER_SIZE - in_idx;
+    }
+    DEBUG(logger->printf("(ar%i/r%i", Serial.available(), maxlen));
+    local_receive_size = Serial.readBytes(temp, maxlen);
+    DEBUG(logger->printf(":r%i/ar%i)\n", local_receive_size, Serial.available()));
+    if (local_receive_size > maxlen) {
+      error("bad read");
+    }
+    if (local_receive_size) {
+      if (memcmp(buf + in_idx, temp, local_receive_size) != 0) {
+        error("fail");
+      }
+      if ((in_idx += local_receive_size) == BUFFER_SIZE) {
+        in_idx = 0;
+      }
+      in_total += local_receive_size;
+    }
+    DEBUG(logger->printf("r(%d) ok\n", local_receive_size));
+  }
+
+  // say something on data every second
+  if ((size_for_led += local_written_size) >= size_for_1sec || millis() > timeout) {
+    digitalWrite(LED_BUILTIN, !digitalRead(LED_BUILTIN));
+    size_for_led = 0;
+
+    if (in_prev == in_total) {
+      error("receiving nothing?\n");
+    }
+
+    unsigned long now_ms = millis();
+    int bwkbps_avg = ((((uint64_t)in_total) * 8000) / (now_ms - start_ms)) >> 10;
+    int bwkbps_now = (((in_total - in_prev) * 8000) / (now_ms - last_ms)) >> 10 ;
+    logger->printf("bwavg=%d bwnow=%d kbps maxavail=%i\n", bwkbps_avg, bwkbps_now, maxavail);
+
+    in_prev = in_total;
+    timeout = (last_ms = now_ms) + TIMEOUT;
+  }
+
+  if (logger->read() == 's') {
+    logger->println("now stopping reading, keeping writing");
+    reading = false;
+  }
+}

package/build_boards_manager_package.sh

@@ -1,6 +1,8 @@
 #!/bin/bash
 #
 
+#set -x
+
 # Extract next version from platform.txt
 next=`sed -n -E 's/version=([0-9.]+)/\1/p' ../platform.txt`
 
@@ -16,6 +18,9 @@ else
     plain_ver=$ver
 fi
 
+# 'set -e' breaks CI but not local tests
+#set -e
+
 package_name=esp8266-$ver
 echo "Version: $ver"
 echo "Package name: $package_name"
@@ -44,10 +49,20 @@ srcdir=$PWD
 rm -rf package/versions/$ver
 mkdir -p $outdir
 
+# Get submodules
+modules=libraries/SoftwareSerial
+for mod in $modules; do
+    echo "refreshing submodule: $mod"
+    git submodule update --init -- $mod
+    (cd $mod && git reset --hard)
+done
+echo "done with submodules"
+
 # Some files should be excluded from the package
 cat << EOF > exclude.txt
 .git
 .gitignore
+.gitmodules
 .travis.yml
 package
 doc
@@ -58,15 +73,6 @@ git ls-files --other --directory >> exclude.txt
 rsync -a --exclude-from 'exclude.txt' $srcdir/ $outdir/
 rm exclude.txt
 
-# Get additional libraries (TODO: add them as git submodule or subtree?)
-
-# SoftwareSerial library
-curl -L -o SoftwareSerial.zip https://github.com/plerup/espsoftwareserial/archive/3.4.1.zip
-unzip -q SoftwareSerial.zip
-rm -rf SoftwareSerial.zip
-mv espsoftwareserial-* SoftwareSerial
-mv SoftwareSerial $outdir/libraries
-
 # For compatibility, on OS X we need GNU sed which is usually called 'gsed'
 if [ "$(uname)" == "Darwin" ]; then
     SED=gsed
@@ -83,6 +89,7 @@ $SED 's/tools.esptool.path={runtime.platform.path}\/tools\/esptool/tools.esptool
 $SED 's/tools.mkspiffs.path={runtime.platform.path}\/tools\/mkspiffs/tools.mkspiffs.path=\{runtime.tools.mkspiffs.path\}/g' |\
 $SED 's/recipe.hooks.core.prebuild.1.pattern.*//g' |\
 $SED 's/recipe.hooks.core.prebuild.2.pattern.*//g' |\
+$SED 's/recipe.hooks.core.prebuild.3.pattern.*//g' |\
 $SED "s/version=.*/version=$ver/g" |\
 $SED -E "s/name=([a-zA-Z0-9\ -]+).*/name=\1($ver)/g"\
  > $outdir/platform.txt
@@ -154,3 +161,5 @@ python ../../merge_packages.py $new_json $old_json >tmp && mv tmp $new_json && r
 
 popd
 popd
+
+echo "All done"

tests/common.sh

@@ -248,6 +248,9 @@ function check_examples_style()
             --suffix=none \
             --options=$TRAVIS_BUILD_DIR/tests/examples_style.conf {} \;
 
+    # we have no control over submodules
+    git submodule foreach --recursive git reset --hard
+
     git diff --exit-code -- $TRAVIS_BUILD_DIR/libraries
 
     echo -e "travis_fold:end:check_examples_style"

tests/device/test_serial/test_serial.ino

@@ -0,0 +1,191 @@
+#include <BSTest.h>
+BS_ENV_DECLARE();
+
+// this is the serialStress.ino example, stripped down
+
+/*
+  Serial read/write/verify/benchmark
+  Using internal loopback
+
+  Released to public domain
+*/
+
+#include <ESP8266WiFi.h>
+
+#define SSBAUD          115200  // console for humans
+#define BAUD            3000000 // hardware serial stress test
+#define BUFFER_SIZE     4096    // may be useless to use more than 2*SERIAL_SIZE_RX
+#define SERIAL_SIZE_RX  1024    // Serial.setRxBufferSize()
+
+#define TIMEOUT 5000
+#define DEBUG(x...) //x
+
+uint8_t buf [BUFFER_SIZE];
+uint8_t temp [BUFFER_SIZE];
+bool reading = true;
+bool overrun = false;
+
+static size_t out_idx = 0, in_idx = 0;
+static size_t local_receive_size = 0;
+static size_t size_for_1sec, size_for_led = 0;
+static size_t maxavail = 0;
+static uint64_t in_total = 0, in_prev = 0;
+static uint64_t start_ms, last_ms;
+static uint64_t timeout;
+
+void preinit() {
+  // (no C++ in function)
+  // disable wifi
+  ESP8266WiFiClass::preinitWiFiOff();
+}
+
+void setup()
+{
+  Serial.begin(SSBAUD);
+
+  int baud = BAUD;
+  size_for_1sec = baud / 10; // 8n1=10baudFor8bits
+  //Serial.printf(ESP.getFullVersion().c_str());
+  //Serial.printf("\n\nBAUD: %d - CoreRxBuffer: %d bytes - TestBuffer: %d bytes\n",
+  //               baud, SERIAL_SIZE_RX, BUFFER_SIZE);
+
+  //Serial.printf("led changes state every %zd bytes (= 1 second)\n", size_for_1sec);
+  //Serial.printf("press 's' to stop reading, not writing (induces overrun)\n");
+
+  BS_RUN(Serial);
+}
+
+void test_setup()
+{
+  Serial.begin(BAUD);
+
+  // bind RX and TX
+  USC0(0) |= (1 << UCLBE);
+
+  Serial.flush();
+  while (Serial.read() != -1);
+  timeout = (start_ms = last_ms = millis()) + TIMEOUT;
+}
+
+void test_setdown ()
+{
+  // unbind RX and TX
+  Serial.flush();
+  USC0(0) &= ~(1 << UCLBE);
+  while (Serial.read() != -1);
+  Serial.begin(SSBAUD);
+}
+
+int bwkbps_avg = 0;
+
+bool test_loop ()
+{
+  size_t maxlen = Serial.availableForWrite();
+  // check remaining space in buffer
+  if (maxlen > BUFFER_SIZE - out_idx) {
+    maxlen = BUFFER_SIZE - out_idx;
+  }
+  // check if not cycling more than buffer size relatively to input
+  size_t in_out = out_idx == in_idx ?
+                  BUFFER_SIZE :
+                  (in_idx + BUFFER_SIZE - out_idx - 1) % BUFFER_SIZE;
+  if (maxlen > in_out) {
+    maxlen = in_out;
+  }
+  size_t local_written_size = Serial.write(buf + out_idx, maxlen);
+  if (local_written_size > maxlen) {
+    return false;
+  }
+  if ((out_idx += local_written_size) == BUFFER_SIZE) {
+    out_idx = 0;
+  }
+  delay(0);
+
+  if (Serial.hasOverrun()) {
+    overrun = true;
+  }
+  if (Serial.hasRxError()) {
+  }
+
+  if (reading)
+  {
+    // receive data
+    maxlen = Serial.available();
+    if (maxlen > maxavail) {
+      maxavail = maxlen;
+    }
+    // check space in temp receive buffer
+    if (maxlen > BUFFER_SIZE - in_idx) {
+      maxlen = BUFFER_SIZE - in_idx;
+    }
+    local_receive_size = Serial.readBytes(temp, maxlen);
+    if (local_receive_size > maxlen) {
+      return false;
+    }
+    if (local_receive_size) {
+      if (memcmp(buf + in_idx, temp, local_receive_size) != 0) {
+        return false;
+      }
+      if ((in_idx += local_receive_size) == BUFFER_SIZE) {
+        in_idx = 0;
+      }
+      in_total += local_receive_size;
+    }
+  }
+
+  // say something on data every second
+  if ((size_for_led += local_written_size) >= size_for_1sec || millis() > timeout) {
+    digitalWrite(LED_BUILTIN, !digitalRead(LED_BUILTIN));
+    size_for_led = 0;
+
+    if (in_prev == in_total) {
+      return false;
+    }
+
+    unsigned long now_ms = millis();
+    bwkbps_avg = ((((uint64_t)in_total) * 8000) / (now_ms - start_ms)) >> 10;
+    //bwkbps_now = (((in_total - in_prev) * 8000) / (now_ms - last_ms)) >> 10 ;
+
+    in_prev = in_total;
+    timeout = (last_ms = now_ms) + TIMEOUT;
+  }
+
+  if (millis() > 5000)
+  {
+    reading = false;
+  }
+  if (millis() > 6000)
+  {
+    return false;
+  }
+  
+  return true;
+}
+
+TEST_CASE("bandwidth and overrun", "[serial]")
+{
+  overrun = false;
+  bwkbps_avg = 0;
+  CHECK(overrun == false);
+  CHECK(bwkbps_avg == 0);
+
+  // let serial flush its BS output before flushing and switching to 3MBPS
+  delay(100);
+  
+  test_setup();
+  while (test_loop());
+  test_setdown();
+
+  Serial.printf("bandwidth = %d kbps  -  overrun=%d\n", bwkbps_avg, overrun);
+
+  // BAUD*10/8/1000 =>kbps *9/10 => 90% at least
+  CHECK(bwkbps_avg > ((((BAUD*8/10)/1000)*9)/10));
+  CHECK(overrun == true);
+
+  while (Serial.read() != -1);
+  Serial.flush();
+}
+
+void loop ()
+{
+}

tests/host/common/MockSerial.cpp

@@ -109,4 +109,11 @@ size_t uart_write (uart_t* uart, const char* buf, size_t size)
 	return write(1, buf, size);
 }
 
+size_t uart_read(uart_t* uart, char* userbuffer, size_t usersize)
+{
+	///XXXTODO
+	(void)uart;
+	return read(0, userbuffer, usersize);
+}
+
 } // extern "C"