1
0
mirror of https://github.com/esp8266/Arduino.git synced 2025-04-19 23:22:16 +03:00

Add FSTools with examples of how to convert between SPIFFS and LITTLEFS. (#7696)

* Add FSTools with examples of how to convert between SPIFFS and LITTLEFS.

* Oops.  Need to pass layout by reference in order to capture the correct address.  Took a while to find that.  There maybe a better way to store all these configs

* Update FSTools.cpp

fix ESP.h to Esp.h

* Fix unused variable i

* Parsed with restyle.sh.  Compile with all errors.

* remove unused variable

* fix different sign complication error

* Fix indentation to spaces
Run test/restyle.sh
Remove commented code
Use #ifdef blocks for debugging.  `DEBUG_ESP_CORE` and `DEBUG_ESP_PORT`
use `static constexpr  layout` which saves ROM ~500B for unused vars

* Update FSTools.cpp

Add yield in between file copy
This commit is contained in:
sticilface 2022-06-26 19:22:31 +01:00 committed by GitHub
parent 5656035bed
commit f0d8f33d83
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 623 additions and 2 deletions

View File

@ -90,9 +90,8 @@ public:
uint8_t obuf[256];
size_t doneLen = 0;
size_t sentLen;
int i;
while (src.available() > sizeof(obuf)){
while (src.available() > (int)sizeof(obuf)){
src.read(obuf, sizeof(obuf));
sentLen = write(obuf, sizeof(obuf));
doneLen = doneLen + sentLen;

View File

@ -0,0 +1,254 @@
#include "FSTools.h"
#include "LittleFS.h"
#include <spiffs_api.h>
#include <Esp.h>
#if defined(DEBUG_ESP_CORE)
#define FSTOOLSDEBUG(_1, ...) { DEBUG_ESP_PORT.printf_P( PSTR(_1),##__VA_ARGS__); }
#else
#define FSTOOLSDEBUG(...) {}
#endif
FSTools::FSTools()
{
}
FSTools::~FSTools()
{
reset();
}
bool FSTools::attemptToMountFS(fs::FS & fs)
{
LittleFSConfig littleFSCfg(false);
SPIFFSConfig SPIFFSCfg(false);
// try to apply the "safe" no format config to the FS... doesn't matter which.. just need one to apply correctly..
if (!fs.setConfig(littleFSCfg) && ! fs.setConfig(SPIFFSCfg))
{
return false;
}
return fs.begin();
}
bool FSTools::mountAlternativeFS(FST::FS_t type, const FST::layout & layout, bool keepMounted)
{
FSConfig * pCfg{nullptr};
LittleFSConfig littleFSCfg(false);
SPIFFSConfig SPIFFSCfg(false);
reset();
switch (type)
{
case FST::SPIFFS :
{
_pFS.reset(new FS(FSImplPtr(new spiffs_impl::SPIFFSImpl(_getStartAddr(layout), _getSize(layout), layout.page, layout.block, 5))));
pCfg = &SPIFFSCfg;
break;
}
case FST::LITTLEFS :
{
_pFS.reset(new FS(FSImplPtr(new littlefs_impl::LittleFSImpl(_getStartAddr(layout), _getSize(layout), layout.page, layout.block, 5))));
pCfg = &littleFSCfg;
break;
}
};
if (_pFS && pCfg && _pFS->setConfig(*pCfg) && _pFS->begin())
{
if (!keepMounted)
{
_pFS->end();
}
_mounted = true;
_layout = &layout;
return true;
}
if (_pFS)
{
_pFS.reset();
}
_mounted = false;
return false;
};
bool FSTools::mounted()
{
return _mounted;
};
bool FSTools::moveFS(fs::FS & destinationFS)
{
uint32_t sourceFileCount = 0;
uint32_t sourceByteTotal = 0;
bool result = false;
if (!_mounted || !_pFS)
{
FSTOOLSDEBUG("Source FS not mounted\n");
return false;
}
uint32_t startSector = (ESP.getSketchSize() + FLASH_SECTOR_SIZE - 1) & (~(FLASH_SECTOR_SIZE - 1));
uint32_t lowestFSStart = 0x40300000;
if (_layout)
{
lowestFSStart = _layout->startAddr;
FSTOOLSDEBUG("_layout->startADDR = 0x%08x\n", _layout->startAddr);
}
uint32_t endSector = lowestFSStart - 0x40200000;
uint32_t tempFSsize = endSector - startSector;
FSTOOLSDEBUG("TempFS: start: %u, end: %u, size: %u, sketchSize = %u, _FS_start = %u\n", startSector, endSector, tempFSsize, ESP.getSketchSize(), (uint32_t)&_FS_start);
fileListIterator(*_pFS, "/", [&sourceFileCount, &sourceByteTotal, this](File & f)
{
if (f)
{
sourceFileCount++;
sourceByteTotal += f.size();
#ifdef DEBUG_ESP_CORE
_dumpFileInfo(f);
#endif
}
});
FSTOOLSDEBUG("%u Files Found Total Size = %u\n", sourceFileCount, sourceByteTotal);
FSTOOLSDEBUG("Size of dummy FS = %u\n", tempFSsize);
FS tempFS = FS(FSImplPtr(new littlefs_impl::LittleFSImpl(startSector, tempFSsize, FS_PHYS_PAGE, FS_PHYS_BLOCK, 5)));
if (tempFS.format() && tempFS.begin())
{
if (_copyFS(*_pFS, tempFS))
{
FSTOOLSDEBUG("Files copied to temp File System\n");
reset();
if (destinationFS.format() && destinationFS.begin()) // must format then mount the new FS
{
if (_copyFS(tempFS, destinationFS))
{
FSTOOLSDEBUG("Files copied back to new FS\n");
result = true;
}
}
else
{
FSTOOLSDEBUG("Error Mounting\n");
}
}
else
{
FSTOOLSDEBUG("Copy Failed\n");
}
tempFS.end();
}
else
{
FSTOOLSDEBUG("Failed to begin() TempFS\n");
}
return result;
};
void FSTools::reset()
{
_mounted = false;
_layout = nullptr;
if (_pFS)
{
_pFS->end();
_pFS.reset();
}
}
void FSTools::fileListIterator(FS & fs, const char * dirName, FST::FileCb Cb)
{
Dir dir = fs.openDir(dirName);
while (dir.next())
{
if (dir.isFile())
{
File f = dir.openFile("r");
if (Cb)
{
Cb(f);
}
}
else
{
fileListIterator(fs, dir.fileName().c_str(), Cb);
}
}
}
uint32_t FSTools::_getStartAddr(const FST::layout & layout)
{
return (layout.startAddr - 0x40200000);
}
uint32_t FSTools::_getSize(const FST::layout & layout)
{
return (layout.endAddr - layout.startAddr);
}
#ifdef DEBUG_ESP_CORE
void FSTools::_dumpFileInfo(File & f)
{
if (f)
{
DEBUG_ESP_PORT.printf_P(PSTR(" File: %-30s [%8uB]\n"), f.fullName(), f.size());
}
}
#endif
bool FSTools::_copyFS(FS & sourceFS, FS & destFS)
{
uint32_t sourceFileCount = 0;
uint32_t sourceByteTotal = 0;
fileListIterator(sourceFS, "/", [&sourceFileCount, &sourceByteTotal](File & f)
{
if (f)
{
sourceFileCount++;
sourceByteTotal += f.size();
}
});
size_t count = 0;
fileListIterator(sourceFS, "/", [&count, &destFS](File & sourceFile)
{
if (sourceFile)
{
File destFile = destFS.open(sourceFile.fullName(), "w");
if (destFile)
{
destFile.setTimeout(5000); // this value was chosen empirically as it failed with default timeout.
size_t written = destFile.write(sourceFile);
if (written == sourceFile.size())
{
count++;
}
}
destFile.close();
sourceFile.close();
yield();
}
});
return (count == sourceFileCount);
}

View File

@ -0,0 +1,91 @@
#pragma once
#include <FS.h>
#include <memory>
#include <functional>
/*
A temporary FS is made between the END of the sketch... and the start of the partition you try to mount, to maximise the available space for copying the FS.
The WORST case this is at 0x40300000 which is for a 3m FS on 4m flash.. leaving 460Kb for copying.
*/
namespace FST
{
struct layout
{
constexpr layout(uint32_t s, uint32_t e, uint32_t p, uint32_t b) : startAddr(s), endAddr(e), page(p), block(b) {};
const uint32_t startAddr{0};
const uint32_t endAddr{0};
const uint32_t page{0};
const uint32_t block{0};
};
enum FS_t : uint8_t
{
SPIFFS,
LITTLEFS
};
static constexpr layout layout_512k32 = { 0x40273000, 0x4027B000, 0x100, 0x1000 };
static constexpr layout layout_512k64 = { 0x4026B000, 0x4027B000, 0x100, 0x1000 };
static constexpr layout layout_512k128 = { 0x4025B000, 0x4027B000, 0x100, 0x1000 };
static constexpr layout layout_1m64 = { 0x402EB000, 0x402FB000, 0x100, 0x1000 };
static constexpr layout layout_1m128 = { 0x402DB000, 0x402FB000, 0x100, 0x1000 };
static constexpr layout layout_1m144 = { 0x402D7000, 0x402FB000, 0x100, 0x1000 };
static constexpr layout layout_1m160 = { 0x402D3000, 0x402FB000, 0x100, 0x1000 };
static constexpr layout layout_1m192 = { 0x402CB000, 0x402FB000, 0x100, 0x1000 };
static constexpr layout layout_1m256 = { 0x402BB000, 0x402FB000, 0x100, 0x1000 };
static constexpr layout layout_1m512 = { 0x4027B000, 0x402FB000, 0x100, 0x2000 };
static constexpr layout layout_2m64 = { 0x403F0000, 0x403FB000, 0x100, 0x1000 };
static constexpr layout layout_2m128 = { 0x403E0000, 0x403FB000, 0x100, 0x1000 };
static constexpr layout layout_2m256 = { 0x403C0000, 0x403FB000, 0x100, 0x1000 };
static constexpr layout layout_2m512 = { 0x40380000, 0x403FA000, 0x100, 0x2000 };
static constexpr layout layout_2m1m = { 0x40300000, 0x403FA000, 0x100, 0x2000 };
static constexpr layout layout_4m1m = { 0x40500000, 0x405FA000, 0x100, 0x2000 };
static constexpr layout layout_4m2m = { 0x40400000, 0x405FA000, 0x100, 0x2000 };
static constexpr layout layout_4m3m = { 0x40300000, 0x405FA000, 0x100, 0x2000 };
static constexpr layout layout_8m6m = { 0x40400000, 0x409FA000, 0x100, 0x2000 };
static constexpr layout layout_8m7m = { 0x40300000, 0x409FA000, 0x100, 0x2000 };
static constexpr layout layout_16m14m = { 0x40400000, 0x411FA000, 0x100, 0x2000 };
static constexpr layout layout_16m15m = { 0x40300000, 0x411FA000, 0x100, 0x2000 };
typedef std::function<void(File & f)> FileCb;
};
class FSTools
{
public:
FSTools();
~FSTools();
bool attemptToMountFS(fs::FS & fs);
bool mountAlternativeFS(FST::FS_t type, const FST::layout & layout, bool keepMounted = false);
bool mounted();
bool moveFS(fs::FS & destinationFS);
void reset();
void fileListIterator(FS & fs, const char * dirName, FST::FileCb Cb);
private:
uint32_t _getStartAddr(const FST::layout & layout);
uint32_t _getSize(const FST::layout & layout);
#ifdef DEBUG_ESP_CORE
void _dumpFileInfo(File & f);
#endif
bool _copyFS(FS & sourceFS, FS & destFS);
std::unique_ptr<fs::FS> _pFS;
bool _mounted{false};
const FST::layout * _layout{nullptr};
};

View File

@ -0,0 +1,130 @@
/*
This sketch will convert SPIFFS partitions to LittleFS on ESP8266
Change the `TARGET_FS_LAYOUT` to the partition layout that you want target
ie what you are trying to copy from.
Include in the sketch whatever you want the destination to be, in this case LittleFS,
but it could be SPIFFS to convert back if need be.
How it works: It creates a LittleFS partition between the end of the sketch and the
start of whatever filesystem you set as target. This has IMPORTANT implications for the
amount of data you can move!!! eg a 4Mb flash module with a 3Mb SPIFFS partition only leaves
about 450k for the temp file system, so if you have more data than that on your 3Mb SPIFFS it
will fail.
*/
#include <ESP8266WiFi.h>
#include <ESP8266mDNS.h>
#include <WiFiUdp.h>
#include <ArduinoOTA.h>
#include <FSTools.h>
#include <LittleFS.h>
#define TARGET_FS_LAYOUT FST::layout_4m3m
FSTools fstools;
#ifndef STASSID
#define STASSID "xxxx"
#define STAPSK "xxxx"
#endif
const char* ssid = STASSID;
const char* password = STAPSK;
bool migrateFS() {
if (!fstools.attemptToMountFS(LittleFS)) { // Attempts to mount LittleFS without autoformat...
Serial.println(F("Default FS not found"));
if (fstools.mountAlternativeFS(FST::SPIFFS /* FST::LITTLEFS */, TARGET_FS_LAYOUT, true)) {
Serial.println(F("Alternative found"));
if (fstools.moveFS(LittleFS)) {
Serial.println(F("FileSystem Moved New FS contents:"));
fstools.fileListIterator(LittleFS, "/", [](File& f) {
Serial.printf_P(PSTR(" File: %-30s [%8uB]\n"), f.fullName(), f.size());
});
return true;
}
}
}
return false;
}
void initWiFiOTA() {
WiFi.mode(WIFI_STA);
WiFi.begin(ssid, password);
while (WiFi.waitForConnectResult() != WL_CONNECTED) {
Serial.println("Connection Failed! Rebooting...");
delay(5000);
ESP.restart();
}
ArduinoOTA.onStart([]() {
String type;
if (ArduinoOTA.getCommand() == U_FLASH) {
type = "sketch";
} else { // U_FS
type = "filesystem";
}
// NOTE: if updating FS this would be the place to unmount FS using FS.end()
Serial.println("Start updating " + type);
});
ArduinoOTA.onEnd([]() {
Serial.println("\nEnd");
});
ArduinoOTA.onProgress([](unsigned int progress, unsigned int total) {
Serial.printf("Progress: %u%%\r", (progress / (total / 100)));
});
ArduinoOTA.onError([](ota_error_t error) {
Serial.printf("Error[%u]: ", error);
if (error == OTA_AUTH_ERROR) {
Serial.println("Auth Failed");
} else if (error == OTA_BEGIN_ERROR) {
Serial.println("Begin Failed");
} else if (error == OTA_CONNECT_ERROR) {
Serial.println("Connect Failed");
} else if (error == OTA_RECEIVE_ERROR) {
Serial.println("Receive Failed");
} else if (error == OTA_END_ERROR) {
Serial.println("End Failed");
}
});
ArduinoOTA.begin();
}
void setup() {
WiFi.persistent(false);
WiFi.disconnect(true);
Serial.begin(115200);
Serial.println();
Serial.printf("SDK Version: %s\n", ESP.getSdkVersion());
Serial.println("Core Version: " + ESP.getCoreVersion());
Serial.println("Full Version: " + ESP.getFullVersion());
Serial.printf("Sketch size: %u\n", ESP.getSketchSize());
Serial.printf("Free size: %u\n", ESP.getFreeSketchSpace());
Serial.println("Booting");
migrateFS(); // MUST call this before calling your own begin();
initWiFiOTA();
Serial.println("Ready");
Serial.print("IP address: ");
Serial.println(WiFi.localIP());
}
void loop() {
ArduinoOTA.handle();
}

View File

@ -0,0 +1,137 @@
/*
This sketch will convert SPIFFS partitions to a custom FS on ESP8266
Change the `TARGET_FS_LAYOUT` to the partition layout that you want target
ie what you are trying to copy from.
This ksetch shows how to create a FS different to the one provided for by the sketch defaults.
This is useful if you need to use an intermediate sketch to move the FS but need to maintain the
sketch size limit of say 512kb.
How it works: It creates a LittleFS partition between the end of the sketch and the
start of whatever filesystem you set as target. This has IMPORTANT implications for the
amount of data you can move!!! eg a 4Mb flash module with a 3Mb SPIFFS partition only leaves
about 450k for the temp file system, so if you have more data than that on your 3Mb SPIFFS it
will fail.
*/
#include <ESP8266WiFi.h>
#include <ESP8266mDNS.h>
#include <WiFiUdp.h>
#include <ArduinoOTA.h>
#include <FSTools.h>
#include <LittleFS.h>
#define TARGET_FS_LAYOUT FST::layout_4m3m
const uint32_t startSector = FST::layout_4m1m.startAddr - 0x40200000;
const uint32_t tempFSsize = FST::layout_4m1m.endAddr - FST::layout_4m1m.startAddr;
fs::FS LittleFS_Different = FS(FSImplPtr(new littlefs_impl::LittleFSImpl(startSector, tempFSsize, FS_PHYS_PAGE, FS_PHYS_BLOCK, 5)));
FSTools fstools;
#ifndef STASSID
#define STASSID "xxxx"
#define STAPSK "xxxx"
#endif
const char* ssid = STASSID;
const char* password = STAPSK;
bool migrateFS() {
if (!fstools.attemptToMountFS(LittleFS_Different)) { // Attempts to mount LittleFS without autoformat...
Serial.println(F("Default FS not found"));
if (fstools.mountAlternativeFS(FST::SPIFFS /* FST::LITTLEFS */, TARGET_FS_LAYOUT, true)) {
Serial.println(F("Alternative found"));
if (fstools.moveFS(LittleFS_Different)) {
Serial.println(F("FileSystem Moved New FS contents:"));
fstools.fileListIterator(LittleFS_Different, "/", [](File& f) {
Serial.printf_P(PSTR(" File: %-30s [%8uB]\n"), f.fullName(), f.size());
});
return true;
}
}
}
return false;
}
void initWiFiOTA() {
WiFi.mode(WIFI_STA);
WiFi.begin(ssid, password);
while (WiFi.waitForConnectResult() != WL_CONNECTED) {
Serial.println("Connection Failed! Rebooting...");
delay(5000);
ESP.restart();
}
ArduinoOTA.onStart([]() {
String type;
if (ArduinoOTA.getCommand() == U_FLASH) {
type = "sketch";
} else { // U_FS
type = "filesystem";
}
// NOTE: if updating FS this would be the place to unmount FS using FS.end()
Serial.println("Start updating " + type);
});
ArduinoOTA.onEnd([]() {
Serial.println("\nEnd");
});
ArduinoOTA.onProgress([](unsigned int progress, unsigned int total) {
Serial.printf("Progress: %u%%\r", (progress / (total / 100)));
});
ArduinoOTA.onError([](ota_error_t error) {
Serial.printf("Error[%u]: ", error);
if (error == OTA_AUTH_ERROR) {
Serial.println("Auth Failed");
} else if (error == OTA_BEGIN_ERROR) {
Serial.println("Begin Failed");
} else if (error == OTA_CONNECT_ERROR) {
Serial.println("Connect Failed");
} else if (error == OTA_RECEIVE_ERROR) {
Serial.println("Receive Failed");
} else if (error == OTA_END_ERROR) {
Serial.println("End Failed");
}
});
ArduinoOTA.begin();
}
void setup() {
WiFi.persistent(false);
WiFi.disconnect(true);
Serial.begin(115200);
Serial.println();
Serial.printf("SDK Version: %s\n", ESP.getSdkVersion());
Serial.println("Core Version: " + ESP.getCoreVersion());
Serial.println("Full Version: " + ESP.getFullVersion());
Serial.printf("Sketch size: %u\n", ESP.getSketchSize());
Serial.printf("Free size: %u\n", ESP.getFreeSketchSpace());
Serial.println("Booting");
migrateFS(); // MUST call this before calling your own begin();
initWiFiOTA();
Serial.println("Ready");
Serial.print("IP address: ");
Serial.println(WiFi.localIP());
}
void loop() {
ArduinoOTA.handle();
}

View File

@ -0,0 +1,10 @@
{
"name": "FSTools",
"keywords": "SPIFFS LittleFS",
"description": "A library that manages convertion between SPIFFS and LITTLEFS as well as mounting partitions outside of sketch default.",
"homepage": "",
"author": "sticilface",
"version": "1.0.0",
"frameworks": "arduino",
"platforms": "esp8266"
}