Move BearSSL from STACK_PROXY to a real, thunked 2nd stack (#5168)

* Update to BearSSL 0.6+ release, add AES_CCM modes

Pull in latest BearSSL head (0.6 + minor additions) release and add AES_CCM
modes to the encryption options.

* Enable the aes_ccm initialization in client/server

* Initial attempt

* Working code with second stack thunking

* Remove #ifdefs in .S file, not needed.

* Clean up thunks and remove separate stack flag

* Fix PIO assembler errors

* Remove #ifdef code changes, ensure same code as PC

Remove "#ifdef ESP8266;...;#else;...;#endif" brackets in BearSSL to
ensure the host-tested code is the same as the ESP8266-run code.

* Move to latest BearSSL w/EC progmem savings

* Merge with master

* Add br_thunk_* calls to do ref counting, painting

Add reference counting br_thunk_add/del_ref() to replace stack handling code
in the class.

Add in stack painting and max usage calculation.

* Add in postmortem stack dump hooks

When a crash occurs while in the second stack, dump the BSSL stack and
then also the stack that it was called from (either cont or sys).

* Update stack dump to match decoder expectations

* Move thunk to code core for linkiage

The thunk code needs to be visible to the core routines, so move it to the
cores/esp8266 directory.  Probably need to refactor the stack setup and the
bearssl portion to avoid dependency on bearssl libs in cores/esp8266

* Add 2nd stack dump utility routine

* Refactor once more, update stack size, add stress

Make stack_thunks generic, remove bearssl include inside of cores/esp8266.

Allocate the stack on a WiFiServerSecure object creation to avoid
fragmentation since we will need to allocate the stack to do any
connected work, anyway.

A stress test is now included which checks the total BearSSL second
stack usage for a variety of TLS handshake and certificate options
from badssl.org.

* Update to latest to-thunks branch

* Add BearSSL device test using stack stress

Run a series of SSL connection and transmission tests that stress
BearSSL and its stack usage to the device tests.

Modify device tests to include a possible SPIFFS generation and
upload when a make_spiffs.py file is present in a test directory.

* Use bearssl/master branch, not /to-thunks branch

Update to use the merged master branch of bearssl.  Should have no code
changes.

cores/esp8266/StackThunk.c

@@ -0,0 +1,122 @@
+/*
+  StackThunk.c - Allow use second stack for BearSSL calls
+
+  BearSSL uses a significant amount of stack space, much larger than
+  the default Arduino core stack. These routines handle swapping
+  between a secondary, user-allocated stack on the heap and the real
+  stack.
+
+  Copyright (c) 2017 Earle F. Philhower, III. All rights reserved.
+
+  This library is free software; you can redistribute it and/or
+  modify it under the terms of the GNU Lesser General Public
+  License as published by the Free Software Foundation; either
+  version 2.1 of the License, or (at your option) any later version.
+
+  This library is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+  Lesser General Public License for more details.
+
+  You should have received a copy of the GNU Lesser General Public
+  License along with this library; if not, write to the Free Software
+  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+  Modified 8 May 2015 by Hristo Gochkov (proper post and file upload handling)
+*/
+
+#include <stdint.h>
+#include <stdlib.h>
+#include "StackThunk.h"
+
+uint32_t *stack_thunk_ptr = NULL;
+uint32_t *stack_thunk_top = NULL;
+uint32_t *stack_thunk_save = NULL;  /* Saved A1 while in BearSSL */
+uint32_t stack_thunk_refcnt = 0;
+
+#define _stackSize (5600/4)
+#define _stackPaint 0xdeadbeef
+
+/* Add a reference, and allocate the stack if necessary */
+void stack_thunk_add_ref()
+{
+  stack_thunk_refcnt++;
+  if (stack_thunk_refcnt == 1) {
+    stack_thunk_ptr = (uint32_t *)malloc(_stackSize * sizeof(uint32_t));
+    stack_thunk_top = stack_thunk_ptr + _stackSize - 1;
+    stack_thunk_save = NULL;
+    stack_thunk_repaint();
+  }
+}
+
+/* Drop a reference, and free stack if no more in use */
+void stack_thunk_del_ref()
+{
+  if (stack_thunk_refcnt == 0) {
+    /* Error! */
+    return;
+  }
+  stack_thunk_refcnt--;
+  if (!stack_thunk_refcnt) {
+    free(stack_thunk_ptr);
+    stack_thunk_ptr = NULL;
+    stack_thunk_top = NULL;
+    stack_thunk_save = NULL;
+  }
+}
+
+void stack_thunk_repaint()
+{
+  for (int i=0; i < _stackSize; i++) {
+    stack_thunk_ptr[i] = _stackPaint;
+  }
+}
+
+/* Simple accessor functions used by postmortem */
+uint32_t stack_thunk_get_refcnt() {
+  return stack_thunk_refcnt;
+}
+
+uint32_t stack_thunk_get_stack_top() {
+  return (uint32_t)stack_thunk_top;
+}
+
+uint32_t stack_thunk_get_stack_bot() {
+  return (uint32_t)stack_thunk_ptr;
+}
+
+uint32_t stack_thunk_get_cont_sp() {
+  return (uint32_t)stack_thunk_save;
+}
+
+/* Return the number of bytes ever used since the stack was created */
+uint32_t stack_thunk_get_max_usage()
+{
+  uint32_t cnt = 0;
+
+  /* No stack == no usage by definition! */
+  if (!stack_thunk_ptr) {
+    return 0;
+  }
+
+  for (cnt=0; (cnt < _stackSize) && (stack_thunk_ptr[cnt] == _stackPaint); cnt++) {
+    /* Noop, all work done in for() */
+  }
+  return 4 * (_stackSize - cnt);
+}
+
+/* Print the stack from the first used 16-byte chunk to the top, decodable by the exception decoder */
+void stack_thunk_dump_stack()
+{
+  uint32_t *pos = stack_thunk_top;
+  while (pos < stack_thunk_ptr) {
+    if ((pos[0] != _stackPaint) || (pos[1] != _stackPaint) || (pos[2] != _stackPaint) || (pos[3] != _stackPaint))
+      break;
+    pos += 4;
+  }
+  ets_printf(">>>stack>>>\n");
+  while (pos < stack_thunk_ptr) {
+    ets_printf("%08x:  %08x %08x %08x %08x\n", pos, pos[0], pos[1], pos[2], pos[3]);
+    pos += 4;
+  }
+  ets_printf("<<<stack<<<\n");
+}

cores/esp8266/StackThunk.h

@@ -0,0 +1,82 @@
+/*
+  StackThunk.h - Allow use second stack for BearSSL calls
+
+  BearSSL uses a significant amount of stack space, much larger than
+  the default Arduino core stack. These routines handle swapping
+  between a secondary, user-allocated stack on the heap and the real
+  stack.
+
+  Copyright (c) 2017 Earle F. Philhower, III. All rights reserved.
+
+  This library is free software; you can redistribute it and/or
+  modify it under the terms of the GNU Lesser General Public
+  License as published by the Free Software Foundation; either
+  version 2.1 of the License, or (at your option) any later version.
+
+  This library is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+  Lesser General Public License for more details.
+
+  You should have received a copy of the GNU Lesser General Public
+  License along with this library; if not, write to the Free Software
+  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+  Modified 8 May 2015 by Hristo Gochkov (proper post and file upload handling)
+*/
+
+#ifndef _STACKTHUNK_H
+#define _STACKTHUNK_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern void stack_thunk_add_ref();
+extern void stack_thunk_del_ref();
+extern void stack_thunk_repaint();
+
+extern uint32_t stack_thunk_get_refcnt();
+extern uint32_t stack_thunk_get_stack_top();
+extern uint32_t stack_thunk_get_stack_bot();
+extern uint32_t stack_thunk_get_cont_sp();
+extern uint32_t stack_thunk_get_max_usage();
+extern void stack_thunk_dump_stack();
+
+// Globals required for thunking operation
+extern uint32_t *stack_thunk_ptr;
+extern uint32_t *stack_thunk_top;
+extern uint32_t *stack_thunk_save;
+extern uint32_t stack_thunk_refcnt;
+
+// Thunking macro
+#define make_stack_thunk(fcnToThunk) \
+__asm("\n\
+.text\n\
+.literal_position\n\
+\n\
+.text\n\
+.global thunk_"#fcnToThunk"\n\
+.type thunk_"#fcnToThunk", @function\n\
+.align 4\n\
+thunk_"#fcnToThunk":\n\
+  addi a1, a1, -16      /* Allocate space for saved registers on stack */\n\
+  s32i a0, a1, 12       /* Store A0, trounced by calls */\n\
+  s32i a15, a1, 8       /* Store A15 (our temporary one) */\n\
+  movi a15, stack_thunk_save  /* Store A1(SP) in temp space */\n\
+  s32i a1, a15, 0\n\
+  movi a15, stack_thunk_top   /* Load A1(SP) with thunk stack */\n\
+  l32i.n a1, a15, 0\n\
+  call0 "#fcnToThunk"      /* Do the call */\n\
+  movi a15, stack_thunk_save  /* Restore A1(SP) */\n\
+  l32i.n a1, a15, 0\n\
+  l32i.n a15, a1, 8     /* Restore the saved registers */\n\
+  l32i.n a0, a1, 12\n\
+  addi a1, a1, 16       /* Free up stack and return to caller */\n\
+  ret\n\
+.size thunk_"#fcnToThunk", . - thunk_"#fcnToThunk"\n");
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

cores/esp8266/core_esp8266_postmortem.c

@@ -29,6 +29,7 @@
 #include "cont.h"
 #include "pgmspace.h"
 #include "gdb_hooks.h"
+#include "StackThunk.h"
 
 extern void __real_system_restart_local();
 
@@ -147,6 +148,17 @@ void __wrap_system_restart_local() {
         offset = 0x10;
     }
 
+    ets_printf_P("\n>>>stack>>>\n");
+
+    if (sp > stack_thunk_get_stack_bot() && sp <= stack_thunk_get_stack_top()) {
+        // BearSSL we dump the BSSL second stack and then reset SP back to the main cont stack
+        ets_printf_P("\nctx: bearssl \n");
+        ets_printf_P("sp: %08x end: %08x offset: %04x\n", sp, stack_thunk_get_stack_top(), offset);
+        print_stack(sp + offset, stack_thunk_get_stack_top());
+        offset = 0; // No offset needed anymore, the exception info was stored in the bssl stack
+        sp = stack_thunk_get_cont_sp();
+    }
+
     if (sp > cont_stack_start && sp < cont_stack_end) {
         ets_printf_P("\nctx: cont \n");
         stack_end = cont_stack_end;
@@ -162,6 +174,8 @@ void __wrap_system_restart_local() {
 
     print_stack(sp + offset, stack_end);
 
+    ets_printf_P("<<<stack<<<\n");
+
     // Use cap-X formatting to ensure the standard EspExceptionDecoder doesn't match the address
     if (umm_last_fail_alloc_addr) {
       ets_printf_P("\nlast failed alloc call: %08X(%d)\n", (uint32_t)umm_last_fail_alloc_addr, umm_last_fail_alloc_size);
@@ -175,7 +189,6 @@ void __wrap_system_restart_local() {
 
 
 static void ICACHE_RAM_ATTR print_stack(uint32_t start, uint32_t end) {
-    ets_printf_P("\n>>>stack>>>\n");
     for (uint32_t pos = start; pos < end; pos += 0x10) {
         uint32_t* values = (uint32_t*)(pos);
 
@@ -185,7 +198,6 @@ static void ICACHE_RAM_ATTR print_stack(uint32_t start, uint32_t end) {
         ets_printf_P("%08x:  %08x %08x %08x %08x %c\n",
             pos, values[0], values[1], values[2], values[3], (looksLikeStackFrame)?'<':' ');
     }
-    ets_printf_P("<<<stack<<<\n");
 }
 
 static void uart_write_char_d(char c) {

libraries/ESP8266WiFi/src/BearSSLHelpers.cpp

@@ -27,6 +27,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include <Arduino.h>
+#include <StackThunk.h>
 #include "BearSSLHelpers.h"
 
 namespace brssl {
@@ -825,5 +826,14 @@ bool X509List::append(const uint8_t *derCert, size_t derLen) {
   return true;
 }
 
-};
+// Second stack thunked helpers
+make_stack_thunk(br_ssl_engine_recvapp_ack);
+make_stack_thunk(br_ssl_engine_recvapp_buf);
+make_stack_thunk(br_ssl_engine_recvrec_ack);
+make_stack_thunk(br_ssl_engine_recvrec_buf);
+make_stack_thunk(br_ssl_engine_sendapp_ack);
+make_stack_thunk(br_ssl_engine_sendapp_buf);
+make_stack_thunk(br_ssl_engine_sendrec_ack);
+make_stack_thunk(br_ssl_engine_sendrec_buf);
 
+};

libraries/ESP8266WiFi/src/BearSSLHelpers.h

@@ -136,6 +136,18 @@ class Session {
     br_ssl_session_parameters _session;
 };
 
+// Stack thunked versions of calls
+extern "C" {
+extern unsigned char *thunk_br_ssl_engine_recvapp_buf( const br_ssl_engine_context *cc, size_t *len);
+extern void thunk_br_ssl_engine_recvapp_ack(br_ssl_engine_context *cc, size_t len);
+extern unsigned char *thunk_br_ssl_engine_recvrec_buf( const br_ssl_engine_context *cc, size_t *len);
+extern void thunk_br_ssl_engine_recvrec_ack(br_ssl_engine_context *cc, size_t len);
+extern unsigned char *thunk_br_ssl_engine_sendapp_buf( const br_ssl_engine_context *cc, size_t *len);
+extern void thunk_br_ssl_engine_sendapp_ack(br_ssl_engine_context *cc, size_t len);
+extern unsigned char *thunk_br_ssl_engine_sendrec_buf( const br_ssl_engine_context *cc, size_t *len);
+extern void thunk_br_ssl_engine_sendrec_ack(br_ssl_engine_context *cc, size_t len);
+};
+
 };
 
 #endif

libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.cpp

@@ -34,6 +34,7 @@ extern "C" {
 #include "ESP8266WiFi.h"
 #include "WiFiClient.h"
 #include "WiFiClientSecureBearSSL.h"
+#include "StackThunk.h"
 #include "lwip/opt.h"
 #include "lwip/ip.h"
 #include "lwip/tcp.h"
@@ -43,15 +44,18 @@ extern "C" {
 #include "c_types.h"
 #include "coredecls.h"
 
+// The BearSSL thunks in use for now
+#define br_ssl_engine_recvapp_ack thunk_br_ssl_engine_recvapp_ack
+#define br_ssl_engine_recvapp_buf thunk_br_ssl_engine_recvapp_buf
+#define br_ssl_engine_recvrec_ack thunk_br_ssl_engine_recvrec_ack
+#define br_ssl_engine_recvrec_buf thunk_br_ssl_engine_recvrec_buf
+#define br_ssl_engine_sendapp_ack thunk_br_ssl_engine_sendapp_ack
+#define br_ssl_engine_sendapp_buf thunk_br_ssl_engine_sendapp_buf
+#define br_ssl_engine_sendrec_ack thunk_br_ssl_engine_sendrec_ack
+#define br_ssl_engine_sendrec_buf thunk_br_ssl_engine_sendrec_buf
+
 namespace BearSSL {
 
-// BearSSL needs a very large stack, larger than the entire ESP8266 Arduino
-// default one.  This shared_pointer is allocated on first use and cleared
-// on last cleanup, with only one stack no matter how many SSL objects.
-std::shared_ptr<uint8_t> WiFiClientSecure::_bearssl_stack = nullptr;
-
-
-
 void WiFiClientSecure::_clear() {
   // TLS handshake may take more than the 5 second default timeout
   _timeout = 15000;
@@ -91,16 +95,7 @@ WiFiClientSecure::WiFiClientSecure() : WiFiClient() {
   _clear();
   _clearAuthenticationSettings();
   _certStore = nullptr; // Don't want to remove cert store on a clear, should be long lived
-  _ensureStackAvailable();
-  _local_bearssl_stack = _bearssl_stack;
-}
-
-void WiFiClientSecure::_ensureStackAvailable() {
-  if (!_bearssl_stack) {
-    const int stacksize = 4500; // Empirically determined stack for EC and RSA connections
-    _bearssl_stack = std::shared_ptr<uint8_t>(new uint8_t[stacksize], std::default_delete<uint8_t[]>());
-    br_esp8266_stack_proxy_init(_bearssl_stack.get(), stacksize);
-  }
+  stack_thunk_add_ref();
 }
 
 WiFiClientSecure::~WiFiClientSecure() {
@@ -110,11 +105,8 @@ WiFiClientSecure::~WiFiClientSecure() {
   }
   free(_cipher_list);
   _freeSSL();
-  _local_bearssl_stack = nullptr;
-  // If there are no other uses than the initial creation, free the stack
-  if (_bearssl_stack.use_count() == 1) {
-    _bearssl_stack = nullptr;
-  }
+  // Serial.printf("Max stack usage: %d bytes\n", br_thunk_get_max_usage());
+  stack_thunk_del_ref();
   if (_deleteChainKeyTA) {
     delete _ta;
     delete _chain;
@@ -127,8 +119,7 @@ WiFiClientSecure::WiFiClientSecure(ClientContext* client,
                                      int iobuf_in_size, int iobuf_out_size, const X509List *client_CA_ta) {
   _clear();
   _clearAuthenticationSettings();
-  _ensureStackAvailable();
-  _local_bearssl_stack = _bearssl_stack;
+  stack_thunk_add_ref();
   _iobuf_in_size = iobuf_in_size;
   _iobuf_out_size = iobuf_out_size;
   _client = client;
@@ -146,8 +137,7 @@ WiFiClientSecure::WiFiClientSecure(ClientContext *client,
                                      int iobuf_in_size, int iobuf_out_size, const X509List *client_CA_ta) {
   _clear();
   _clearAuthenticationSettings();
-  _ensureStackAvailable();
-  _local_bearssl_stack = _bearssl_stack;
+  stack_thunk_add_ref();
   _iobuf_in_size = iobuf_in_size;
   _iobuf_out_size = iobuf_out_size;
   _client = client;

libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.h

@@ -227,13 +227,6 @@ class WiFiClientSecure : public WiFiClient {
 
     // AXTLS compatible mode needs to delete the stored certs and keys on destruction
     bool _deleteChainKeyTA;
-
-  private:
-    // Single memory buffer used for BearSSL auxilliary stack, insead of growing main Arduino stack for all apps
-    static std::shared_ptr<uint8_t> _bearssl_stack;
-    void _ensureStackAvailable(); // Allocate the stack if necessary
-    // The local copy, only used to enable a reference count
-    std::shared_ptr<uint8_t> _local_bearssl_stack;
 };
 
 };

libraries/ESP8266WiFi/src/WiFiServerSecureBearSSL.cpp

@@ -26,6 +26,7 @@ extern "C" {
 #include "ets_sys.h"
 }
 
+#include <StackThunk.h>
 #include "debug.h"
 #include "ESP8266WiFi.h"
 #include "WiFiClient.h"
@@ -40,14 +41,17 @@ namespace BearSSL {
 
 // Only need to call the standard server constructor
 WiFiServerSecure::WiFiServerSecure(IPAddress addr, uint16_t port) : WiFiServer(addr, port) {
+  stack_thunk_add_ref();
 }
 
 // Only need to call the standard server constructor
 WiFiServerSecure::WiFiServerSecure(uint16_t port) : WiFiServer(port) {
+  stack_thunk_add_ref();
 }
 
 // Destructor only checks if we need to delete compatibilty cert/key
 WiFiServerSecure::~WiFiServerSecure() {
+  stack_thunk_del_ref();
   if (_deleteChainAndKey) {
     delete _chain;
     delete _sk;

tests/README.md

@@ -114,6 +114,7 @@ Some tests need to connect to WiFi AP or to the PC running the tests. In the tes
 
 Environment variables can also be used to pass some information from the test code to the host side helper. To do that, test code can set an environment variable using `setenv` C function. Then the `teardown` host side helper can obtain the value of that variable using `request_env` function defined in `mock_decorators`.
 
+A SPIFFS filesystem may be generated on the host and uploade before a test by including a file called `make_spiffs.py` in the individual test directory.
 
 ### Building and running the tests
 

tests/device/Makefile

@@ -5,6 +5,7 @@ ESP8266_CORE_PATH ?= ../..
 BUILD_DIR ?= $(PWD)/.build
 HARDWARE_DIR ?= $(PWD)/.hardware
 ESPTOOL ?= $(ESP8266_CORE_PATH)/tools/esptool/esptool
+MKSPIFFS ?= $(ESP8266_CORE_PATH)/tools/mkspiffs/mkspiffs
 UPLOAD_PORT ?= $(shell ls /dev/tty* | grep -m 1 -i USB)
 UPLOAD_BAUD ?= 921600
 UPLOAD_BOARD ?= nodemcu
@@ -55,6 +56,16 @@ ifneq ("$(NO_BUILD)","1")
 endif
 ifneq ("$(NO_UPLOAD)","1")
 	@test -n "$(UPLOAD_PORT)" || (echo "Failed to detect upload port, please export UPLOAD_PORT manually" && exit 1)
+	@test -e $(dir $@)/make_spiffs.py && (echo "Generating and uploading SPIFFS" && \
+		(cd $(dir $@) && python ./make_spiffs.py) && \
+		$(MKSPIFFS) --create $(dir $@)data/ --size 0xFB000 \
+			--block 8192 --page 256 $(LOCAL_BUILD_DIR)/spiffs.img && \
+		$(ESPTOOL) $(UPLOAD_VERBOSE_FLAG) \
+			-cp $(UPLOAD_PORT) \
+			-cb $(UPLOAD_BAUD) \
+			-cd $(UPLOAD_BOARD) \
+			-ca 0x300000 \
+			-cf $(LOCAL_BUILD_DIR)/spiffs.img ) || (echo "No SPIFFS to upload")
 	@echo Uploading binary
 	$(SILENT)$(ESPTOOL) $(UPLOAD_VERBOSE_FLAG)  \
 		-cp $(UPLOAD_PORT) \

tests/device/libraries/BSTest/runner.py

@@ -132,7 +132,7 @@ class BSTestRunner(object):
 
     def run_test(self, index):
         self.sp.sendline('{}'.format(index))
-        timeout = 10
+        timeout = 20 # 10
         while timeout > 0:
             res = self.sp.expect(['>>>>>bs_test_start', EOF, TIMEOUT])
             if res == 0:

tests/device/test_BearSSL/make_spiffs.py

@@ -0,0 +1,66 @@
+#!/usr/bin/python
+
+# This script pulls the list of Mozilla trusted certificate authorities
+# from the web at the "mozurl" below, parses the file to grab the PEM
+# for each cert, and then generates DER files in a new ./data directory
+# Upload these to a SPIFFS filesystem and use the CertManager to parse
+# and use them for your outgoing SSL connections.
+#
+# Script by Earle F. Philhower, III.  Released to the public domain.
+
+import csv
+import os
+from subprocess import Popen, PIPE, call
+import urllib2
+try:
+    # for Python 2.x
+    from StringIO import StringIO
+except ImportError:
+    # for Python 3.x
+    from io import StringIO
+
+# Mozilla's URL for the CSV file with included PEM certs
+mozurl = "https://ccadb-public.secure.force.com/mozilla/IncludedCACertificateReportPEMCSV"
+
+# Load the manes[] and pems[] array from the URL
+names = []
+pems = []
+response = urllib2.urlopen(mozurl)
+csvData = response.read()
+csvReader = csv.reader(StringIO(csvData))
+for row in csvReader:
+    names.append(row[0]+":"+row[1]+":"+row[2])
+    pems.append(row[28])
+del names[0] # Remove headers
+del pems[0] # Remove headers
+
+# Try and make ./data, skip if present
+try:
+    os.mkdir("data")
+except:
+    pass
+
+derFiles = []
+idx = 0
+# Process the text PEM using openssl into DER files
+for i in range(0, len(pems)):
+    certName = "data/ca_%03d.der" % (idx);
+    thisPem = pems[i].replace("'", "")
+    print names[i] + " -> " + certName
+    ssl = Popen(['openssl','x509','-inform','PEM','-outform','DER','-out', certName], shell = False, stdin = PIPE)
+    pipe = ssl.stdin
+    pipe.write(thisPem)
+    pipe.close()
+    ssl.wait()
+    if os.path.exists(certName):
+        derFiles.append(certName)
+        idx = idx + 1
+
+if os.path.exists("data/certs.ar"):
+    os.unlink("data/certs.ar");
+
+arCmd = ['ar', 'q', 'data/certs.ar'] + derFiles;
+call( arCmd )
+
+for der in derFiles:
+    os.unlink(der)

tests/device/test_BearSSL/test_BearSSL.ino

@@ -0,0 +1,220 @@
+// Stress test the BearSSL connection options to determine
+// maximum memory use for different SSL connections and
+// SPIFFS certstore usage.  Before running you need to run
+// certs-from-mozilla.py and upload the generated SPIFFS file.
+//
+// For more info on CertStores, see the BearSSL_CertStore example
+//
+// November 2018 by Earle F. Philhower, III
+// Released to the public domain
+
+#include <Arduino.h>
+#include <BSTest.h>
+#include <ESP8266WiFi.h>
+#include <CertStoreBearSSL.h>
+#include <FS.h>
+#include <time.h>
+#include <StackThunk.h>
+
+extern "C" {
+#include "user_interface.h"
+}
+
+BS_ENV_DECLARE();
+
+void setClock();
+
+// A single, global CertStore which can be used by all
+// connections.  Needs to stay live the entire time any of
+// the WiFiClientBearSSLs are present.
+BearSSL::CertStore certStore;
+
+// NOTE: The CertStoreFile virtual class may migrate to a templated
+// model in a future release. Expect some changes to the interface,
+// no matter what, as the SD and SPIFFS filesystem get unified.
+
+class SPIFFSCertStoreFile : public BearSSL::CertStoreFile {
+  public:
+    SPIFFSCertStoreFile(const char *name) {
+      _name = name;
+    };
+    virtual ~SPIFFSCertStoreFile() override {};
+
+    // The main API
+    virtual bool open(bool write = false) override {
+      _file = SPIFFS.open(_name, write ? "w" : "r");
+      return _file;
+    }
+    virtual bool seek(size_t absolute_pos) override {
+      return _file.seek(absolute_pos, SeekSet);
+    }
+    virtual ssize_t read(void *dest, size_t bytes) override {
+      return _file.readBytes((char*)dest, bytes);
+    }
+    virtual ssize_t write(void *dest, size_t bytes) override {
+      return _file.write((uint8_t*)dest, bytes);
+    }
+    virtual void close() override {
+      _file.close();
+    }
+
+  private:
+    File _file;
+    const char *_name;
+};
+
+SPIFFSCertStoreFile certs_idx("/certs.idx"); // Generated by the ESP8266
+SPIFFSCertStoreFile certs_ar("/certs.ar"); // Uploaded by the user
+
+
+void setup()
+{
+    Serial.begin(115200);
+    Serial.setDebugOutput(true);
+    WiFi.persistent(false);
+    WiFi.mode(WIFI_STA);
+    WiFi.begin(getenv("STA_SSID"), getenv("STA_PASS"));
+    while (WiFi.status() != WL_CONNECTED) {
+        delay(500);
+    }
+    setClock();
+    SPIFFS.begin();
+    int numCerts = certStore.initCertStore(&certs_idx, &certs_ar);
+    Serial.printf("Number of CA certs read: %d\n", numCerts);
+    if (numCerts == 0) {
+        Serial.printf("No certs found. Did you run certs-from-mozill.py and upload the SPIFFS directory before running?\n");
+        REQUIRE(1==0);
+    }
+    BS_RUN(Serial);
+}
+
+static void stopAll()
+{
+    WiFiClient::stopAll();
+}
+
+static void disconnectWiFI()
+{
+    wifi_station_disconnect();
+}
+
+
+// Set time via NTP, as required for x.509 validation
+void setClock() {
+  configTime(3 * 3600, 0, "pool.ntp.org", "time.nist.gov");
+
+  Serial.print("Waiting for NTP time sync: ");
+  time_t now = time(nullptr);
+  while (now < 8 * 3600 * 2) {
+    delay(500);
+    Serial.print(".");
+    now = time(nullptr);
+  }
+  Serial.println("");
+  struct tm timeinfo;
+  gmtime_r(&now, &timeinfo);
+  Serial.print("Current time: ");
+  Serial.print(asctime(&timeinfo));
+}
+
+// Try and connect using a WiFiClientBearSSL to specified host:port and dump URL
+void fetchURL(BearSSL::WiFiClientSecure *client, const char *host, const uint16_t port, const char *path) {
+  if (!path) {
+    path = "/";
+  }
+
+  Serial.printf("Trying: %s:443...", host);
+  client->connect(host, port);
+  if (!client->connected()) {
+    Serial.printf("*** Can't connect. ***\n-------\n");
+    return;
+  }
+  Serial.printf("Connected!\n-------\n");
+  client->write("GET ");
+  client->write(path);
+  client->write(" HTTP/1.0\r\nHost: ");
+  client->write(host);
+  client->write("\r\nUser-Agent: ESP8266\r\n");
+  client->write("\r\n");
+  uint32_t to = millis() + 5000;
+  if (client->connected()) {
+    do {
+      char tmp[32];
+      memset(tmp, 0, 32);
+      int rlen = client->read((uint8_t*)tmp, sizeof(tmp) - 1);
+      yield();
+      if (rlen < 0) {
+        break;
+      }
+      // Only print out first line up to \r, then abort connection
+      char *nl = strchr(tmp, '\r');
+      if (nl) {
+        *nl = 0;
+        Serial.print(tmp);
+        break;
+      }
+      Serial.print(tmp);
+    } while (millis() < to);
+  }
+  client->stop();
+  Serial.printf("\n-------\n");
+}
+
+
+int run(const char *str)
+{
+    BearSSL::WiFiClientSecure *bear = new BearSSL::WiFiClientSecure();
+    // Integrate the cert store with this connection
+    bear->setCertStore(&certStore);
+
+    char buff[100];
+    uint32_t maxUsage = 0;
+    stack_thunk_repaint();
+    sprintf(buff, "%s.badssl.com", str);
+    Serial.printf("%s: ", buff);
+    fetchURL(bear, buff, 443, "/");
+    Serial.printf("Stack: %d\n", stack_thunk_get_max_usage());
+    maxUsage = std::max(maxUsage, stack_thunk_get_max_usage());
+    delete bear;
+
+    printf("\n\n\nMAX THUNK STACK USAGE: %d\n", maxUsage);
+    return maxUsage;
+}
+
+#define TC(x) TEST_CASE("BearSSL - Maximum stack usage < 5600 bytes @ "x".badssl.org", "[bearssl]") { REQUIRE(run(x) < 5600); }
+
+TC("expired")
+TC("wrong.host")
+TC("self-signed")
+TC("untrusted-root")
+TC("revoked")
+TC("pinning-test")
+TC("no-common-name")
+TC("no-subject")
+TC("incomplete-chain")
+TC("sha1-intermediate")
+TC("sha256")
+TC("sha384")
+TC("sha512")
+TC("1000-sans")
+// TC("10000-sans") // Runs for >10 seconds, so causes false failure.  Covered by the 1000 SAN anyway
+TC("ecc256")
+TC("ecc384")
+TC("rsa2048")
+TC("rsa4096")
+TC("extended-validation")
+TC("dh480")
+TC("dh512")
+TC("dh1024")
+TC("dh2048")
+TC("dh-small-subgroup")
+TC("dh-composite")
+TC("static-rsa")
+TC("tls-v1-0")
+TC("tls-v1-1")
+TC("tls-v1-2")
+TC("invalid-expected-sct")
+
+void loop() {
+}
+

tools/sdk/include/bearssl/bearssl.h

@@ -136,7 +136,6 @@
 #include "bearssl_ssl.h"
 #include "bearssl_x509.h"
 #include "bearssl_pem.h"
-#include "bearssl_port.h"
 
 /** \brief Type for a configuration option.
  *

tools/sdk/include/bearssl/bearssl_git.h

@@ -1,2 +1,2 @@
 // Do not edit -- Automatically generated by tools/sdk/ssl/bearssl/Makefile
-#define BEARSSL_GIT f55a6ad
+#define BEARSSL_GIT 0e71e46