From 8859b818d8557ebada8d60e76891ac7e23f5aa96 Mon Sep 17 00:00:00 2001 From: "Earle F. Philhower, III" Date: Thu, 30 May 2019 12:53:03 -0700 Subject: [PATCH] Make CertStore natively use File interface (#6131) __This is a breaking change, but the header and example did warn everyone that this API was in flux due to the incompatible SD and SPIFFS File implementations.__ BearSSL CertStores now simply need a filesystem and the names of the data (generated on-chip) and archive (uploaded by user) files on it. No more need to roll your own virtual CertStoreFile class. Update the library, examples, and device test. --- .../BearSSL_CertStore/BearSSL_CertStore.ino | 94 ++----------------- .../ESP8266WiFi/src/CertStoreBearSSL.cpp | 82 +++++++++------- libraries/ESP8266WiFi/src/CertStoreBearSSL.h | 33 ++----- .../httpUpdateSecure/httpUpdateSecure.ino | 38 +------- tests/device/Makefile | 6 +- tests/device/test_BearSSL/make_spiffs.py | 2 +- tests/device/test_BearSSL/test_BearSSL.ino | 40 +------- 7 files changed, 70 insertions(+), 225 deletions(-) diff --git a/libraries/ESP8266WiFi/examples/BearSSL_CertStore/BearSSL_CertStore.ino b/libraries/ESP8266WiFi/examples/BearSSL_CertStore/BearSSL_CertStore.ino index 37c73c67e..20a8a445a 100644 --- a/libraries/ESP8266WiFi/examples/BearSSL_CertStore/BearSSL_CertStore.ino +++ b/libraries/ESP8266WiFi/examples/BearSSL_CertStore/BearSSL_CertStore.ino @@ -36,6 +36,7 @@ #include #include #include +#include #ifndef STASSID #define STASSID "your-ssid" @@ -50,87 +51,6 @@ const char *pass = STAPSK; // the WiFiClientBearSSLs are present. BearSSL::CertStore certStore; -// Uncomment below to use the SD card to store the certs -// #define USE_SDCARD 1 - -// NOTE: The CertStoreFile virtual class may migrate to a templated -// model in a future release. Expect some changes to the interface, -// no matter what, as the SD and SPIFFS filesystem get unified. - -#ifdef USE_SDCARD - -#include -class SDCertStoreFile : public BearSSL::CertStoreFile { - public: - SDCertStoreFile(const char *name) { - _name = name; - }; - virtual ~SDCertStoreFile() override {}; - - // The main API - virtual bool open(bool write = false) override { - _file = SD.open(_name, write ? FILE_WRITE : FILE_READ); - return _file; - } - virtual bool seek(size_t absolute_pos) override { - return _file.seek(absolute_pos); - } - virtual ssize_t read(void *dest, size_t bytes) override { - return _file.read(dest, bytes); - } - virtual ssize_t write(void *dest, size_t bytes) override { - return _file.write((const uint8_t*)dest, bytes); - } - virtual void close() override { - _file.close(); - } - - private: - File _file; - const char *_name; -}; - -SDCertStoreFile certs_idx("/certs.idx"); // Generated by the ESP8266 -SDCertStoreFile certs_ar("/certs.ar"); // Uploaded by the user - -#else - -#include -class SPIFFSCertStoreFile : public BearSSL::CertStoreFile { - public: - SPIFFSCertStoreFile(const char *name) { - _name = name; - }; - virtual ~SPIFFSCertStoreFile() override {}; - - // The main API - virtual bool open(bool write = false) override { - _file = SPIFFS.open(_name, write ? "w" : "r"); - return _file; - } - virtual bool seek(size_t absolute_pos) override { - return _file.seek(absolute_pos, SeekSet); - } - virtual ssize_t read(void *dest, size_t bytes) override { - return _file.readBytes((char*)dest, bytes); - } - virtual ssize_t write(void *dest, size_t bytes) override { - return _file.write((uint8_t*)dest, bytes); - } - virtual void close() override { - _file.close(); - } - - private: - File _file; - const char *_name; -}; - -SPIFFSCertStoreFile certs_idx("/certs.idx"); // Generated by the ESP8266 -SPIFFSCertStoreFile certs_ar("/certs.ar"); // Uploaded by the user - -#endif - // Set time via NTP, as required for x.509 validation void setClock() { configTime(3 * 3600, 0, "pool.ntp.org", "time.nist.gov"); @@ -197,11 +117,8 @@ void setup() { Serial.println(); Serial.println(); -#ifdef USE_SDCARD - SD.begin(); -#else SPIFFS.begin(); -#endif + // If using a SD card or LittleFS, call the appropriate ::begin instead // We start by connecting to a WiFi network Serial.print("Connecting to "); @@ -221,10 +138,10 @@ void setup() { setClock(); // Required for X.509 validation - int numCerts = certStore.initCertStore(&certs_idx, &certs_ar); + int numCerts = certStore.initCertStore(SPIFFS, PSTR("/certs.idx"), PSTR("/certs.ar")); Serial.printf("Number of CA certs read: %d\n", numCerts); if (numCerts == 0) { - Serial.printf("No certs found. Did you run certs-from-mozill.py and upload the SPIFFS directory before running?\n"); + Serial.printf("No certs found. Did you run certs-from-mozilla.py and upload the SPIFFS directory before running?\n"); return; // Can't connect to anything w/o certs! } @@ -242,6 +159,9 @@ void loop() { do { site = Serial.readString(); } while (site == ""); + // Strip newline if present + site.replace(String("\r"), emptyString); + site.replace(String("\n"), emptyString); Serial.printf("https://%s/\n", site.c_str()); BearSSL::WiFiClientSecure *bear = new BearSSL::WiFiClientSecure(); diff --git a/libraries/ESP8266WiFi/src/CertStoreBearSSL.cpp b/libraries/ESP8266WiFi/src/CertStoreBearSSL.cpp index dbcb6e103..8affc7e01 100644 --- a/libraries/ESP8266WiFi/src/CertStoreBearSSL.cpp +++ b/libraries/ESP8266WiFi/src/CertStoreBearSSL.cpp @@ -37,6 +37,12 @@ extern "C" { } } + +CertStore::~CertStore() { + free(_indexName); + free(_dataName); +} + CertStore::CertInfo CertStore::_preprocessCert(uint32_t length, uint32_t offset, const void *raw) { CertStore::CertInfo ci; @@ -70,46 +76,54 @@ CertStore::CertInfo CertStore::_preprocessCert(uint32_t length, uint32_t offset, // The certs.ar file is a UNIX ar format file, concatenating all the // individual certificates into a single blob in a space-efficient way. -int CertStore::initCertStore(CertStoreFile *index, CertStoreFile *data) { +int CertStore::initCertStore(FS &fs, const char *indexFileName, const char *dataFileName) { int count = 0; uint32_t offset = 0; - _index = index; - _data = data; + _fs = &fs; - if (!_index || !data) { + // No strdup_P, so manually do it + _indexName = (char *)malloc(strlen_P(indexFileName) + 1); + _dataName = (char *)malloc(strlen_P(dataFileName) + 1); + if (!_indexName || !_dataName) { + free(_indexName); + free(_dataName); + return 0; + } + memcpy_P(_indexName, indexFileName, strlen_P(indexFileName) + 1); + memcpy_P(_dataName, dataFileName, strlen_P(dataFileName) + 1); + + File index = _fs->open(_indexName, "w"); + if (!index) { return 0; } - if (!_index->open(true)) { + File data = _fs->open(_dataName, "r"); + if (!data) { + index.close(); return 0; } - if (!_data->open(false)) { - _index->close(); - return 0; - } - - char magic[8]; - if (_data->read(magic, sizeof(magic)) != sizeof(magic) || + uint8_t magic[8]; + if (data.read(magic, sizeof(magic)) != sizeof(magic) || memcmp(magic, "!\n", sizeof(magic)) ) { - _data->close(); - _index->close(); + data.close(); + index.close(); return 0; } offset += sizeof(magic); while (true) { - char fileHeader[60]; + uint8_t fileHeader[60]; // 0..15 = filename in ASCII // 48...57 = length in decimal ASCII uint32_t length; - if (data->read(fileHeader, sizeof(fileHeader)) != sizeof(fileHeader)) { + if (data.read(fileHeader, sizeof(fileHeader)) != sizeof(fileHeader)) { break; } offset += sizeof(fileHeader); fileHeader[58] = 0; - if (1 != sscanf(fileHeader + 48, "%d", &length) || !length) { + if (1 != sscanf((char *)(fileHeader + 48), "%d", &length) || !length) { break; } @@ -117,7 +131,7 @@ int CertStore::initCertStore(CertStoreFile *index, CertStoreFile *data) { if (!raw) { break; } - if (_data->read(raw, length) != (ssize_t)length) { + if (data.read((uint8_t *)raw, length) != length) { free(raw); break; } @@ -125,7 +139,7 @@ int CertStore::initCertStore(CertStoreFile *index, CertStoreFile *data) { // If the filename starts with "//" then this is a rename file, skip it if (fileHeader[0] != '/' || fileHeader[1] != '/') { CertStore::CertInfo ci = _preprocessCert(length, offset, raw); - if (_index->write(&ci, sizeof(ci)) != (ssize_t)sizeof(ci)) { + if (index.write((uint8_t *)&ci, sizeof(ci)) != (ssize_t)sizeof(ci)) { free(raw); break; } @@ -135,13 +149,13 @@ int CertStore::initCertStore(CertStoreFile *index, CertStoreFile *data) { offset += length; free(raw); if (offset & 1) { - char x; - _data->read(&x, 1); + uint8_t x; + data.read(&x, 1); offset++; } } - _data->close(); - _index->close(); + data.close(); + index.close(); return count; } @@ -153,35 +167,37 @@ const br_x509_trust_anchor *CertStore::findHashedTA(void *ctx, void *hashed_dn, CertStore *cs = static_cast(ctx); CertStore::CertInfo ci; - if (!cs || len != sizeof(ci.sha256) || !cs->_index || !cs->_data) { + if (!cs || len != sizeof(ci.sha256) || !cs->_indexName || !cs->_dataName || !cs->_fs) { return nullptr; } - if (!cs->_index->open(false)) { + File index = cs->_fs->open(cs->_indexName, "r"); + if (!index) { return nullptr; } - while (cs->_index->read(&ci, sizeof(ci)) == sizeof(ci)) { + while (index.read((uint8_t *)&ci, sizeof(ci)) == sizeof(ci)) { if (!memcmp(ci.sha256, hashed_dn, sizeof(ci.sha256))) { - cs->_index->close(); + index.close(); uint8_t *der = (uint8_t*)malloc(ci.length); if (!der) { return nullptr; } - if (!cs->_data->open(false)) { + File data = cs->_fs->open(cs->_dataName, "r"); + if (!data) { free(der); return nullptr; } - if (!cs->_data->seek(ci.offset)) { - cs->_data->close(); + if (!data.seek(ci.offset, SeekSet)) { + data.close(); free(der); return nullptr; } - if (cs->_data->read(der, ci.length) != (ssize_t)ci.length) { + if (data.read((uint8_t *)der, ci.length) != ci.length) { free(der); return nullptr; } - cs->_data->close(); + data.close(); cs->_x509 = new X509List(der, ci.length); free(der); if (!cs->_x509) { @@ -196,7 +212,7 @@ const br_x509_trust_anchor *CertStore::findHashedTA(void *ctx, void *hashed_dn, return ta; } } - cs->_index->close(); + index.close(); return nullptr; } diff --git a/libraries/ESP8266WiFi/src/CertStoreBearSSL.h b/libraries/ESP8266WiFi/src/CertStoreBearSSL.h index cc5a3432e..76d4966e6 100644 --- a/libraries/ESP8266WiFi/src/CertStoreBearSSL.h +++ b/libraries/ESP8266WiFi/src/CertStoreBearSSL.h @@ -23,6 +23,7 @@ #include #include #include +#include // Base class for the certificate stores, which allow use // of a large set of certificates stored on SPIFFS of SD card to @@ -30,43 +31,21 @@ namespace BearSSL { -// Subclass this and provide virtual functions appropriate for your storage. -// Required because there are conflicting definitions for a "File" in the -// Arduino setup, and there is no simple way to work around the minor -// differences. -// See the examples for implementations to use in your own code. -// -// NOTE: This virtual class may migrate to a templated model in a future -// release. Expect some changes to the interface, no matter what, as the -// SD and SPIFFS filesystem get unified. -class CertStoreFile { - public: - CertStoreFile() {}; - virtual ~CertStoreFile() {}; - - // The main API - virtual bool open(bool write=false) = 0; - virtual bool seek(size_t absolute_pos) = 0; - virtual ssize_t read(void *dest, size_t bytes) = 0; - virtual ssize_t write(void *dest, size_t bytes) = 0; - virtual void close() = 0; -}; - - class CertStore { public: CertStore() { }; - ~CertStore() { }; + ~CertStore(); // Set the file interface instances, do preprocessing - int initCertStore(CertStoreFile *index, CertStoreFile *data); + int initCertStore(FS &fs, const char *indexFileName, const char *dataFileName); // Installs the cert store into the X509 decoder (normally via static function callbacks) void installCertStore(br_x509_minimal_context *ctx); protected: - CertStoreFile *_index = nullptr; - CertStoreFile *_data = nullptr; + FS *_fs = nullptr; + char *_indexName = nullptr; + char *_dataName = nullptr; X509List *_x509 = nullptr; // These need to be static as they are callbacks from BearSSL C code diff --git a/libraries/ESP8266httpUpdate/examples/httpUpdateSecure/httpUpdateSecure.ino b/libraries/ESP8266httpUpdate/examples/httpUpdateSecure/httpUpdateSecure.ino index e50ef54f5..9eba1e514 100644 --- a/libraries/ESP8266httpUpdate/examples/httpUpdateSecure/httpUpdateSecure.ino +++ b/libraries/ESP8266httpUpdate/examples/httpUpdateSecure/httpUpdateSecure.ino @@ -13,6 +13,8 @@ #include +#include + #define USE_SERIAL Serial #ifndef APSSID @@ -28,40 +30,6 @@ ESP8266WiFiMulti WiFiMulti; #include BearSSL::CertStore certStore; -#include -class SPIFFSCertStoreFile : public BearSSL::CertStoreFile { - public: - SPIFFSCertStoreFile(const char *name) { - _name = name; - }; - virtual ~SPIFFSCertStoreFile() override {}; - - // The main API - virtual bool open(bool write = false) override { - _file = SPIFFS.open(_name, write ? "w" : "r"); - return _file; - } - virtual bool seek(size_t absolute_pos) override { - return _file.seek(absolute_pos, SeekSet); - } - virtual ssize_t read(void *dest, size_t bytes) override { - return _file.readBytes((char*)dest, bytes); - } - virtual ssize_t write(void *dest, size_t bytes) override { - return _file.write((uint8_t*)dest, bytes); - } - virtual void close() override { - _file.close(); - } - - private: - File _file; - const char *_name; -}; - -SPIFFSCertStoreFile certs_idx("/certs.idx"); -SPIFFSCertStoreFile certs_ar("/certs.ar"); - // Set time via NTP, as required for x.509 validation void setClock() { configTime(0, 0, "pool.ntp.org", "time.nist.gov"); // UTC @@ -102,7 +70,7 @@ void setup() { SPIFFS.begin(); - int numCerts = certStore.initCertStore(&certs_idx, &certs_ar); + int numCerts = certStore.initCertStore(SPIFFS, PSTR("/certs.idx"), PSTR("/certs.ar")); USE_SERIAL.print(F("Number of CA certs read: ")); USE_SERIAL.println(numCerts); if (numCerts == 0) { USE_SERIAL.println(F("No certs found. Did you run certs-from-mozill.py and upload the SPIFFS directory before running?")); diff --git a/tests/device/Makefile b/tests/device/Makefile index 8c9385648..2d2c80fa8 100644 --- a/tests/device/Makefile +++ b/tests/device/Makefile @@ -1,7 +1,7 @@ SHELL := /bin/bash V ?= 0 TEST_LIST ?= $(wildcard test_*/*.ino) -ESP8266_CORE_PATH ?= ../.. +ESP8266_CORE_PATH ?= $(realpath ../..) BUILD_DIR ?= $(PWD)/.build HARDWARE_DIR ?= $(PWD)/.hardware #PYTHON ?= python3 @@ -60,9 +60,9 @@ ifneq ("$(NO_UPLOAD)","1") @test -n "$(UPLOAD_PORT)" || (echo "Failed to detect upload port, please export UPLOAD_PORT manually" && exit 1) @test -e $(dir $@)/make_spiffs.py && (echo "Generating and uploading SPIFFS" && \ (cd $(dir $@) && $(PYTHON) ./make_spiffs.py) && \ - $(SILENT)$(MKSPIFFS) --create $(dir $@)data/ --size 0xFB000 \ + $(MKSPIFFS) --create $(dir $@)data/ --size 0xFB000 \ --block 8192 --page 256 $(LOCAL_BUILD_DIR)/spiffs.img && \ - $(SILENT)$(ESPTOOL) $(UPLOAD_VERBOSE_FLAG) \ + $(ESPTOOL) $(UPLOAD_VERBOSE_FLAG) \ --chip esp8266 \ --port $(UPLOAD_PORT) \ --baud $(UPLOAD_BAUD) \ diff --git a/tests/device/test_BearSSL/make_spiffs.py b/tests/device/test_BearSSL/make_spiffs.py index 049df11a2..13c77ccf5 100755 --- a/tests/device/test_BearSSL/make_spiffs.py +++ b/tests/device/test_BearSSL/make_spiffs.py @@ -30,7 +30,7 @@ csvData = response.read() csvReader = csv.reader(StringIO(csvData)) for row in csvReader: names.append(row[0]+":"+row[1]+":"+row[2]) - pems.append(row[28]) + pems.append(row[30]) del names[0] # Remove headers del pems[0] # Remove headers diff --git a/tests/device/test_BearSSL/test_BearSSL.ino b/tests/device/test_BearSSL/test_BearSSL.ino index 1530dc6ca..675c24de6 100644 --- a/tests/device/test_BearSSL/test_BearSSL.ino +++ b/tests/device/test_BearSSL/test_BearSSL.ino @@ -29,44 +29,6 @@ void setClock(); // the WiFiClientBearSSLs are present. BearSSL::CertStore certStore; -// NOTE: The CertStoreFile virtual class may migrate to a templated -// model in a future release. Expect some changes to the interface, -// no matter what, as the SD and SPIFFS filesystem get unified. - -class SPIFFSCertStoreFile : public BearSSL::CertStoreFile { - public: - SPIFFSCertStoreFile(const char *name) { - _name = name; - }; - virtual ~SPIFFSCertStoreFile() override {}; - - // The main API - virtual bool open(bool write = false) override { - _file = SPIFFS.open(_name, write ? "w" : "r"); - return _file; - } - virtual bool seek(size_t absolute_pos) override { - return _file.seek(absolute_pos, SeekSet); - } - virtual ssize_t read(void *dest, size_t bytes) override { - return _file.readBytes((char*)dest, bytes); - } - virtual ssize_t write(void *dest, size_t bytes) override { - return _file.write((uint8_t*)dest, bytes); - } - virtual void close() override { - _file.close(); - } - - private: - File _file; - const char *_name; -}; - -SPIFFSCertStoreFile certs_idx("/certs.idx"); // Generated by the ESP8266 -SPIFFSCertStoreFile certs_ar("/certs.ar"); // Uploaded by the user - - void setup() { Serial.begin(115200); @@ -79,7 +41,7 @@ void setup() } setClock(); SPIFFS.begin(); - int numCerts = certStore.initCertStore(&certs_idx, &certs_ar); + int numCerts = certStore.initCertStore(SPIFFS, "/certs.idx", "/certs.ar"); Serial.printf("Number of CA certs read: %d\n", numCerts); if (numCerts == 0) { Serial.printf("No certs found. Did you run certs-from-mozill.py and upload the SPIFFS directory before running?\n");