1
0
mirror of https://github.com/esp8266/Arduino.git synced 2025-06-29 16:03:14 +03:00

Remove dependency on SD/SPIFFS from CertStore (#4760)

Due to popular demand, remove the hardcoded dependency on SPIFFS
or SD from the CertStore by factoring out the file interface into
a new class (CertStoreFile) that the user will need to implement
as a thin wrapper around either a SPIFFS.file or a SD.file

Combine the downloaded certificates into a UNIX "ar" archive
and parse that on-the-fly to allow easy inspection and creation
of the Cert Store database.

Examples updated with a new certificate downloader that creates
the certs.ar archive and with a single sample that can be built
for either SPIFFS or SD with a #define.  Users can copy the
implementation of the CertStoreFile they need to their own code
as it is self-contained.

Also move the CertStore to the BearSSL namespace and remove the
suffix and separate SPIFFS/SD sources.

Remove the "deep+" change from the CI build as well (no special
options needed on any PIO or makefile build).

We'll revisit the filesystem wrapper for 2.5.0, hopefully having a
unified template for both filesystem usage at a global level.  For
current users, be aware the interface may change (simplify!) in
release 2.5.0.

Fixes #4740
This commit is contained in:
Earle F. Philhower, III
2018-06-12 19:06:26 -07:00
committed by GitHub
parent c0cfe875c2
commit 794630e068
10 changed files with 296 additions and 469 deletions

View File

@ -20,21 +20,18 @@
#include "CertStoreBearSSL.h"
#include <memory>
namespace BearSSL {
extern "C" {
// Callbacks for the x509 decoder
// Callback for the x509 decoder
static void dn_append(void *ctx, const void *buf, size_t len) {
br_sha256_context *sha1 = (br_sha256_context*)ctx;
br_sha256_update(sha1, buf, len);
}
static void dn_append_null(void *ctx, const void *buf, size_t len) {
(void) ctx;
(void) buf;
(void) len;
}
}
CertStoreBearSSL::CertInfo CertStoreBearSSL::preprocessCert(const char *fname, const void *raw, size_t sz) {
CertStoreBearSSL::CertInfo ci;
CertStore::CertInfo CertStore::_preprocessCert(uint32_t length, uint32_t offset, const void *raw) {
CertStore::CertInfo ci;
// Clear the CertInfo
memset(&ci, 0, sizeof(ci));
@ -44,11 +41,12 @@ CertStoreBearSSL::CertInfo CertStoreBearSSL::preprocessCert(const char *fname, c
br_sha256_context *sha256 = new br_sha256_context;
br_sha256_init(sha256);
br_x509_decoder_init(ctx, dn_append, sha256, nullptr, nullptr);
br_x509_decoder_push(ctx, (const void*)raw, sz);
br_x509_decoder_push(ctx, (const void*)raw, length);
// Copy result to structure
br_sha256_out(sha256, &ci.sha256);
strcpy(ci.fname, fname);
ci.length = length;
ci.offset = offset;
// Clean up allocated memory
delete sha256;
@ -58,84 +56,139 @@ CertStoreBearSSL::CertInfo CertStoreBearSSL::preprocessCert(const char *fname, c
return ci;
}
br_x509_trust_anchor *CertStoreBearSSL::makeTrustAnchor(const void *der, size_t der_len, const CertInfo *ci) {
// std::unique_ptr will free dc when we exit scope, automatically
std::unique_ptr<br_x509_decoder_context> dc(new br_x509_decoder_context);
br_x509_decoder_init(dc.get(), dn_append_null, nullptr, nullptr, nullptr);
br_x509_decoder_push(dc.get(), der, der_len);
br_x509_pkey *pk = br_x509_decoder_get_pkey(dc.get());
if (!pk) {
return nullptr;
// The certs.ar file is a UNIX ar format file, concatenating all the
// individual certificates into a single blob in a space-efficient way.
int CertStore::initCertStore(CertStoreFile *index, CertStoreFile *data) {
int count = 0;
uint32_t offset = 0;
_index = index;
_data = data;
if (!_index || !data) {
return 0;
}
br_x509_trust_anchor *ta = (br_x509_trust_anchor*)malloc(sizeof(br_x509_trust_anchor));
if (!ta) {
return nullptr;
}
memset(ta, 0, sizeof(*ta));
ta->dn.data = (uint8_t*)malloc(sizeof(ci->sha256));
if (!ta->dn.data) {
free(ta);
return nullptr;
}
memcpy(ta->dn.data, ci->sha256, sizeof(ci->sha256));
ta->dn.len = sizeof(ci->sha256);
ta->flags = 0;
if (br_x509_decoder_isCA(dc.get())) {
ta->flags |= BR_X509_TA_CA;
if (!_index->open(true)) {
return 0;
}
switch (pk->key_type) {
case BR_KEYTYPE_RSA:
ta->pkey.key_type = BR_KEYTYPE_RSA;
ta->pkey.key.rsa.n = (uint8_t*)malloc(pk->key.rsa.nlen);
if (!ta->pkey.key.rsa.n) {
free(ta->dn.data);
free(ta);
return nullptr;
}
memcpy(ta->pkey.key.rsa.n, pk->key.rsa.n, pk->key.rsa.nlen);
ta->pkey.key.rsa.nlen = pk->key.rsa.nlen;
ta->pkey.key.rsa.e = (uint8_t*)malloc(pk->key.rsa.elen);
if (!ta->pkey.key.rsa.e) {
free(ta->pkey.key.rsa.n);
free(ta->dn.data);
free(ta);
return nullptr;
}
memcpy(ta->pkey.key.rsa.e, pk->key.rsa.e, pk->key.rsa.elen);
ta->pkey.key.rsa.elen = pk->key.rsa.elen;
return ta;
case BR_KEYTYPE_EC:
ta->pkey.key_type = BR_KEYTYPE_EC;
ta->pkey.key.ec.curve = pk->key.ec.curve;
ta->pkey.key.ec.q = (uint8_t*)malloc(pk->key.ec.qlen);
if (!ta->pkey.key.ec.q) {
free(ta->dn.data);
free(ta);
return nullptr;
}
memcpy(ta->pkey.key.ec.q, pk->key.ec.q, pk->key.ec.qlen);
ta->pkey.key.ec.qlen = pk->key.ec.qlen;
return ta;
default:
free(ta->dn.data);
free(ta);
return nullptr;
if (!_data->open(false)) {
_index->close();
return 0;
}
char magic[8];
if (_data->read(magic, sizeof(magic)) != sizeof(magic) ||
memcmp(magic, "!<arch>\n", sizeof(magic)) ) {
_data->close();
_index->close();
return 0;
}
offset += sizeof(magic);
while (true) {
char fileHeader[60];
// 0..15 = filename in ASCII
// 48...57 = length in decimal ASCII
uint32_t length;
if (data->read(fileHeader, sizeof(fileHeader)) != sizeof(fileHeader)) {
break;
}
offset += sizeof(fileHeader);
fileHeader[58] = 0;
if (1 != sscanf(fileHeader + 48, "%d", &length) || !length) {
break;
}
void *raw = malloc(length);
if (!raw) {
break;
}
if (_data->read(raw, length) != (ssize_t)length) {
free(raw);
break;
}
// If the filename starts with "//" then this is a rename file, skip it
if (fileHeader[0] != '/' || fileHeader[1] != '/') {
CertStore::CertInfo ci = _preprocessCert(length, offset, raw);
if (_index->write(&ci, sizeof(ci)) != (ssize_t)sizeof(ci)) {
free(raw);
break;
}
count++;
}
offset += length;
free(raw);
if (offset & 1) {
char x;
_data->read(&x, 1);
offset++;
}
}
_data->close();
_index->close();
return count;
}
void CertStore::installCertStore(br_x509_minimal_context *ctx) {
br_x509_minimal_set_dynamic(ctx, (void*)this, findHashedTA, freeHashedTA);
}
const br_x509_trust_anchor *CertStore::findHashedTA(void *ctx, void *hashed_dn, size_t len) {
CertStore *cs = static_cast<CertStore*>(ctx);
CertStore::CertInfo ci;
if (!cs || len != sizeof(ci.sha256) || !cs->_index || !cs->_data) {
return nullptr;
}
if (!cs->_index->open(false)) {
return nullptr;
}
while (cs->_index->read(&ci, sizeof(ci)) == sizeof(ci)) {
if (!memcmp(ci.sha256, hashed_dn, sizeof(ci.sha256))) {
cs->_index->close();
uint8_t *der = (uint8_t*)malloc(ci.length);
if (!der) {
return nullptr;
}
if (!cs->_data->open(false)) {
free(der);
return nullptr;
}
if (!cs->_data->seek(ci.offset)) {
cs->_data->close();
free(der);
return nullptr;
}
if (cs->_data->read(der, ci.length) != (ssize_t)ci.length) {
free(der);
return nullptr;
}
cs->_data->close();
cs->_x509 = new BearSSLX509List(der, ci.length);
free(der);
br_x509_trust_anchor *ta = (br_x509_trust_anchor*)cs->_x509->getTrustAnchors();
memcpy(ta->dn.data, ci.sha256, sizeof(ci.sha256));
ta->dn.len = sizeof(ci.sha256);
return ta;
}
}
cs->_index->close();
return nullptr;
}
void CertStore::freeHashedTA(void *ctx, const br_x509_trust_anchor *ta) {
CertStore *cs = static_cast<CertStore*>(ctx);
(void) ta; // Unused
delete cs->_x509;
cs->_x509 = nullptr;
}
void CertStoreBearSSL::freeTrustAnchor(const br_x509_trust_anchor *ta) {
switch (ta->pkey.key_type) {
case BR_KEYTYPE_RSA:
free(ta->pkey.key.rsa.e);
free(ta->pkey.key.rsa.n);
break;
case BR_KEYTYPE_EC:
free(ta->pkey.key.ec.q);
break;
}
free(ta->dn.data);
free((void*)ta);
}

View File

@ -21,40 +21,69 @@
#define _CERTSTORE_BEARSSL_H
#include <Arduino.h>
#include <BearSSLHelpers.h>
#include <bearssl/bearssl.h>
// Virtual base class for the certificate stores, which allow use
// Base class for the certificate stores, which allow use
// of a large set of certificates stored on SPIFFS of SD card to
// be dynamically used when validating a X509 certificate
// Templates for child classes not possible due to the difference in SD
// and FS in terms of directory parsing and interating. Dir doesn't
// exist in SD, everything is a file (which might support get-next-entry()
// or not).
namespace BearSSL {
// This class should not be instantiated directly, only via its children.
class CertStoreBearSSL {
// Subclass this and provide virtual functions appropriate for your storage.
// Required because there are conflicting definitions for a "File" in the
// Arduino setup, and there is no simple way to work around the minor
// differences.
// See the examples for implementations to use in your own code.
//
// NOTE: This virtual class may migrate to a templated model in a future
// release. Expect some changes to the interface, no matter what, as the
// SD and SPIFFS filesystem get unified.
class CertStoreFile {
public:
CertStoreBearSSL() {}
virtual ~CertStoreBearSSL() {}
CertStoreFile() {};
virtual ~CertStoreFile() {};
// Preprocess the certs from the flash, returns number parsed
virtual int initCertStore(const char *dir) = 0;
// The main API
virtual bool open(bool write=false) = 0;
virtual bool seek(size_t absolute_pos) = 0;
virtual ssize_t read(void *dest, size_t bytes) = 0;
virtual ssize_t write(void *dest, size_t bytes) = 0;
virtual void close() = 0;
};
class CertStore {
public:
CertStore() { };
~CertStore() { };
// Set the file interface instances, do preprocessing
int initCertStore(CertStoreFile *index, CertStoreFile *data);
// Installs the cert store into the X509 decoder (normally via static function callbacks)
virtual void installCertStore(br_x509_minimal_context *ctx) = 0;
void installCertStore(br_x509_minimal_context *ctx);
protected:
// The binary format of the pre-computed file
CertStoreFile *_index = nullptr;
CertStoreFile *_data = nullptr;
BearSSLX509List *_x509 = nullptr;
// These need to be static as they are callbacks from BearSSL C code
static const br_x509_trust_anchor *findHashedTA(void *ctx, void *hashed_dn, size_t len);
static void freeHashedTA(void *ctx, const br_x509_trust_anchor *ta);
// The binary format of the index file
class CertInfo {
public:
uint8_t sha256[32];
char fname[64];
uint32_t offset;
uint32_t length;
};
static CertInfo _preprocessCert(uint32_t length, uint32_t offset, const void *raw);
};
CertInfo preprocessCert(const char *fname, const void *raw, size_t sz);
static br_x509_trust_anchor *makeTrustAnchor(const void *der, size_t der_len, const CertInfo *ci);
static void freeTrustAnchor(const br_x509_trust_anchor *ta);
};
#endif

View File

@ -1,141 +0,0 @@
/*
CertStoreSDBearSSL.cpp - Library for Arduino ESP8266
Copyright (c) 2018 Earle F. Philhower, III
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include <SD.h>
#include "CertStoreSDBearSSL.h"
CertStoreSDBearSSL::CertStoreSDBearSSL() : CertStoreBearSSL() {
path = "";
}
CertStoreSDBearSSL::~CertStoreSDBearSSL() {
}
CertStoreBearSSL::CertInfo CertStoreSDBearSSL::preprocessCert(File *f) {
CertStoreBearSSL::CertInfo ci;
memset(&ci, 0, sizeof(ci));
// Load the DER into RAM temporarially
if (!f) {
return ci;
}
int sz = f->size();
uint8_t *buf = new uint8_t[sz];
if (!buf) {
return ci;
}
f->read(buf, sz);
ci = CertStoreBearSSL::preprocessCert(f->name(), buf, sz);
delete buf;
return ci;
}
int CertStoreSDBearSSL::initCertStore(const char *subdir) {
int count = 0;
// We want path to have a leading slash and a trailing one
path = subdir;
if (path[0] != '/') {
path = "/" + path;
}
if (!path.endsWith("/")) {
path += "/";
}
String tblName = path + "ca_tbl.bin";
File tbl = SD.open(tblName, FILE_WRITE);
if (!tbl) {
return 0;
}
File d = SD.open(path);
while (true) {
File nextFile = d.openNextFile();
if (!nextFile) {
break;
}
if (!strstr(nextFile.name(), ".der")) {
continue;
}
CertStoreBearSSL::CertInfo ci = preprocessCert(&nextFile);
nextFile.close();
tbl.write((uint8_t*)&ci, sizeof(ci));
count++;
}
tbl.close();
return count;
}
void CertStoreSDBearSSL::installCertStore(br_x509_minimal_context *ctx) {
br_x509_minimal_set_dynamic(ctx, (void*)this, findHashedTA, freeHashedTA);
}
const br_x509_trust_anchor *CertStoreSDBearSSL::findHashedTA(void *ctx, void *hashed_dn, size_t len) {
CertStoreSDBearSSL *cs = static_cast<CertStoreSDBearSSL*>(ctx);
CertInfo ci;
String tblName = cs->path + "ca_tbl.bin";
if (len != sizeof(ci.sha256) || !SD.exists(tblName)) {
return nullptr;
}
File f = SD.open(tblName, FILE_READ);
if (!f) {
return nullptr;
}
while (f.read((uint8_t*)&ci, sizeof(ci)) == sizeof(ci)) {
if (!memcmp(ci.sha256, hashed_dn, sizeof(ci.sha256))) {
// This could be the one!
f.close();
File d = SD.open(ci.fname, FILE_READ);
if (!d) {
return nullptr;
}
size_t der_len = d.size();
uint8_t *der = (uint8_t*)malloc(der_len);
if (!der) {
d.close();
return nullptr;
}
if (d.read(der, der_len) != (int)der_len) {
d.close();
free(der);
return nullptr;
}
d.close();
br_x509_trust_anchor *ta = CertStoreBearSSL::makeTrustAnchor(der, der_len, &ci);
free(der);
return ta;
}
}
f.close();
return nullptr;
}
void CertStoreSDBearSSL::freeHashedTA(void *ctx, const br_x509_trust_anchor *ta) {
(void) ctx; // not needed
CertStoreBearSSL::freeTrustAnchor(ta);
}

View File

@ -1,47 +0,0 @@
/*
CertStoreSDBearSSL.h - Library for Arduino ESP8266
Copyright (c) 2018 Earle F. Philhower, III
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*/
#ifndef _CERTSTORESD_BEARSSL_H
#define _CERTSTORESD_BEARSSL_H
#include "CertStoreBearSSL.h"
class File;
// SD cert store can be in a subdirectory as there are fewer limits
// Note that SD.begin() MUST be called before doing initCertStore because
// there are different options for the CS and other pins you need to
// specify it in your own code.
class CertStoreSDBearSSL : public CertStoreBearSSL {
public:
CertStoreSDBearSSL();
virtual ~CertStoreSDBearSSL();
virtual int initCertStore(const char *dir = "/") override;
virtual void installCertStore(br_x509_minimal_context *ctx) override;
private:
String path;
CertInfo preprocessCert(File *f);
// These need to be static as they are callbacks from BearSSL C code
static const br_x509_trust_anchor *findHashedTA(void *ctx, void *hashed_dn, size_t len);
static void freeHashedTA(void *ctx, const br_x509_trust_anchor *ta);
};
#endif

View File

@ -1,125 +0,0 @@
/*
CertStoreSPIFFSBearSSL.cpp - Library for Arduino ESP8266
Copyright (c) 2018 Earle F. Philhower, III
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "CertStoreSPIFFSBearSSL.h"
#include <FS.h>
CertStoreSPIFFSBearSSL::CertStoreSPIFFSBearSSL() : CertStoreBearSSL() {
}
CertStoreSPIFFSBearSSL::~CertStoreSPIFFSBearSSL() {
}
CertStoreBearSSL::CertInfo CertStoreSPIFFSBearSSL::preprocessCert(const char *fname) {
CertStoreBearSSL::CertInfo ci;
memset(&ci, 0, sizeof(ci));
// Load the DER into RAM temporarially
File f = SPIFFS.open(fname, "r");
if (!f) {
return ci;
}
int sz = f.size();
uint8_t *buf = new uint8_t[sz];
if (!buf) {
f.close();
return ci;
}
f.read(buf, sz);
f.close();
ci = CertStoreBearSSL::preprocessCert(fname, buf, sz);
delete[] buf;
return ci;
}
int CertStoreSPIFFSBearSSL::initCertStore(const char *subdir) {
(void) subdir; // ignored prefix, not enough space in filenames
int count = 0;
SPIFFS.begin();
File tbl = SPIFFS.open("/ca_tbl.bin", "w");
if (!tbl) {
return 0;
}
Dir d = SPIFFS.openDir("");
while (d.next()) {
if (!strstr(d.fileName().c_str(), ".der")) {
continue;
}
CertStoreBearSSL::CertInfo ci = preprocessCert(d.fileName().c_str());
tbl.write((uint8_t*)&ci, sizeof(ci));
count++;
}
tbl.close();
return count;
}
void CertStoreSPIFFSBearSSL::installCertStore(br_x509_minimal_context *ctx) {
br_x509_minimal_set_dynamic(ctx, /* no context needed */nullptr, findHashedTA, freeHashedTA);
}
const br_x509_trust_anchor *CertStoreSPIFFSBearSSL::findHashedTA(void *ctx, void *hashed_dn, size_t len) {
(void) ctx; // not needed
CertInfo ci;
if (len != sizeof(ci.sha256) || !SPIFFS.exists("/ca_tbl.bin")) {
return nullptr;
}
File f = SPIFFS.open("/ca_tbl.bin", "r");
if (!f) {
return nullptr;
}
while (f.read((uint8_t*)&ci, sizeof(ci)) == sizeof(ci)) {
if (!memcmp(ci.sha256, hashed_dn, sizeof(ci.sha256))) {
// This could be the one!
f.close();
File d = SPIFFS.open(ci.fname, "r");
if (!d) {
return nullptr;
}
size_t der_len = d.size();
uint8_t *der = (uint8_t*)malloc(der_len);
if (!der) {
d.close();
return nullptr;
}
if (d.read(der, der_len) != der_len) {
d.close();
free(der);
return nullptr;
}
d.close();
br_x509_trust_anchor *ta = CertStoreBearSSL::makeTrustAnchor(der, der_len, &ci);
free(der);
return ta;
}
}
f.close();
return nullptr;
}
void CertStoreSPIFFSBearSSL::freeHashedTA(void *ctx, const br_x509_trust_anchor *ta) {
(void) ctx; // not needed
CertStoreBearSSL::freeTrustAnchor(ta);
}

View File

@ -1,43 +0,0 @@
/*
CertStoreSPIFFSBearSSL.h - Library for Arduino ESP8266
Copyright (c) 2018 Earle F. Philhower, III
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*/
#ifndef _CERTSTORESPIFFS_BEARSSL_H
#define _CERTSTORESPIFFS_BEARSSL_H
#include "CertStoreBearSSL.h"
#include <FS.h>
// SPIFFS cert stores stored in root directory due to filename length limits
class CertStoreSPIFFSBearSSL : public CertStoreBearSSL {
public:
CertStoreSPIFFSBearSSL();
virtual ~CertStoreSPIFFSBearSSL();
virtual int initCertStore(const char *dir = "") override; // ignores dir
virtual void installCertStore(br_x509_minimal_context *ctx) override;
private:
CertInfo preprocessCert(const char *fname);
// These need to be static as they are callbacks from BearSSL C code
static const br_x509_trust_anchor *findHashedTA(void *ctx, void *hashed_dn, size_t len);
static void freeHashedTA(void *ctx, const br_x509_trust_anchor *ta);
};
#endif

View File

@ -95,7 +95,7 @@ class WiFiClientSecure : public WiFiClient {
int getLastSSLError(char *dest = NULL, size_t len = 0);
// Attach a preconfigured certificate store
void setCertStore(CertStoreBearSSL *certStore) {
void setCertStore(CertStore *certStore) {
_certStore = certStore;
}
@ -152,7 +152,7 @@ class WiFiClientSecure : public WiFiClient {
std::shared_ptr<unsigned char> _iobuf_out;
time_t _now;
const BearSSLX509List *_ta;
CertStoreBearSSL *_certStore;
CertStore *_certStore;
int _iobuf_in_size;
int _iobuf_out_size;
bool _handshake_done;