From f0d8f33d83d14eca2f7a88024f4fffd1efb4d399 Mon Sep 17 00:00:00 2001 From: sticilface Date: Sun, 26 Jun 2022 19:22:31 +0100 Subject: [PATCH] Add FSTools with examples of how to convert between SPIFFS and LITTLEFS. (#7696) * Add FSTools with examples of how to convert between SPIFFS and LITTLEFS. * Oops. Need to pass layout by reference in order to capture the correct address. Took a while to find that. There maybe a better way to store all these configs * Update FSTools.cpp fix ESP.h to Esp.h * Fix unused variable i * Parsed with restyle.sh. Compile with all errors. * remove unused variable * fix different sign complication error * Fix indentation to spaces Run test/restyle.sh Remove commented code Use #ifdef blocks for debugging. `DEBUG_ESP_CORE` and `DEBUG_ESP_PORT` use `static constexpr layout` which saves ROM ~500B for unused vars * Update FSTools.cpp Add yield in between file copy --- cores/esp8266/FS.h | 3 +- libraries/FSTools/FSTools.cpp | 254 ++++++++++++++++++ libraries/FSTools/FSTools.h | 91 +++++++ .../examples/Basic_example/Basic_example.ino | 130 +++++++++ .../custom_FS_example/custom_FS_example.ino | 137 ++++++++++ libraries/FSTools/package.json | 10 + 6 files changed, 623 insertions(+), 2 deletions(-) create mode 100644 libraries/FSTools/FSTools.cpp create mode 100644 libraries/FSTools/FSTools.h create mode 100644 libraries/FSTools/examples/Basic_example/Basic_example.ino create mode 100644 libraries/FSTools/examples/custom_FS_example/custom_FS_example.ino create mode 100644 libraries/FSTools/package.json diff --git a/cores/esp8266/FS.h b/cores/esp8266/FS.h index 709932107..55305e968 100644 --- a/cores/esp8266/FS.h +++ b/cores/esp8266/FS.h @@ -90,9 +90,8 @@ public: uint8_t obuf[256]; size_t doneLen = 0; size_t sentLen; - int i; - while (src.available() > sizeof(obuf)){ + while (src.available() > (int)sizeof(obuf)){ src.read(obuf, sizeof(obuf)); sentLen = write(obuf, sizeof(obuf)); doneLen = doneLen + sentLen; diff --git a/libraries/FSTools/FSTools.cpp b/libraries/FSTools/FSTools.cpp new file mode 100644 index 000000000..c18e968b7 --- /dev/null +++ b/libraries/FSTools/FSTools.cpp @@ -0,0 +1,254 @@ +#include "FSTools.h" +#include "LittleFS.h" +#include +#include + + + +#if defined(DEBUG_ESP_CORE) +#define FSTOOLSDEBUG(_1, ...) { DEBUG_ESP_PORT.printf_P( PSTR(_1),##__VA_ARGS__); } +#else +#define FSTOOLSDEBUG(...) {} +#endif + + +FSTools::FSTools() +{ + +} + +FSTools::~FSTools() +{ + reset(); +} + + +bool FSTools::attemptToMountFS(fs::FS & fs) +{ + LittleFSConfig littleFSCfg(false); + SPIFFSConfig SPIFFSCfg(false); + // try to apply the "safe" no format config to the FS... doesn't matter which.. just need one to apply correctly.. + if (!fs.setConfig(littleFSCfg) && ! fs.setConfig(SPIFFSCfg)) + { + return false; + } + return fs.begin(); +} + + +bool FSTools::mountAlternativeFS(FST::FS_t type, const FST::layout & layout, bool keepMounted) +{ + FSConfig * pCfg{nullptr}; + LittleFSConfig littleFSCfg(false); + SPIFFSConfig SPIFFSCfg(false); + reset(); + + switch (type) + { + case FST::SPIFFS : + { + _pFS.reset(new FS(FSImplPtr(new spiffs_impl::SPIFFSImpl(_getStartAddr(layout), _getSize(layout), layout.page, layout.block, 5)))); + pCfg = &SPIFFSCfg; + break; + } + case FST::LITTLEFS : + { + _pFS.reset(new FS(FSImplPtr(new littlefs_impl::LittleFSImpl(_getStartAddr(layout), _getSize(layout), layout.page, layout.block, 5)))); + pCfg = &littleFSCfg; + break; + } + }; + + if (_pFS && pCfg && _pFS->setConfig(*pCfg) && _pFS->begin()) + { + if (!keepMounted) + { + _pFS->end(); + } + _mounted = true; + _layout = &layout; + return true; + } + + if (_pFS) + { + _pFS.reset(); + } + _mounted = false; + return false; +}; + + +bool FSTools::mounted() +{ + return _mounted; +}; + + +bool FSTools::moveFS(fs::FS & destinationFS) +{ + uint32_t sourceFileCount = 0; + uint32_t sourceByteTotal = 0; + bool result = false; + + if (!_mounted || !_pFS) + { + FSTOOLSDEBUG("Source FS not mounted\n"); + return false; + } + + uint32_t startSector = (ESP.getSketchSize() + FLASH_SECTOR_SIZE - 1) & (~(FLASH_SECTOR_SIZE - 1)); + uint32_t lowestFSStart = 0x40300000; + + if (_layout) + { + lowestFSStart = _layout->startAddr; + FSTOOLSDEBUG("_layout->startADDR = 0x%08x\n", _layout->startAddr); + } + + uint32_t endSector = lowestFSStart - 0x40200000; + uint32_t tempFSsize = endSector - startSector; + + FSTOOLSDEBUG("TempFS: start: %u, end: %u, size: %u, sketchSize = %u, _FS_start = %u\n", startSector, endSector, tempFSsize, ESP.getSketchSize(), (uint32_t)&_FS_start); + + fileListIterator(*_pFS, "/", [&sourceFileCount, &sourceByteTotal, this](File & f) + { + if (f) + { + sourceFileCount++; + sourceByteTotal += f.size(); + +#ifdef DEBUG_ESP_CORE + _dumpFileInfo(f); +#endif + + } + }); + + FSTOOLSDEBUG("%u Files Found Total Size = %u\n", sourceFileCount, sourceByteTotal); + FSTOOLSDEBUG("Size of dummy FS = %u\n", tempFSsize); + + FS tempFS = FS(FSImplPtr(new littlefs_impl::LittleFSImpl(startSector, tempFSsize, FS_PHYS_PAGE, FS_PHYS_BLOCK, 5))); + + if (tempFS.format() && tempFS.begin()) + { + if (_copyFS(*_pFS, tempFS)) + { + FSTOOLSDEBUG("Files copied to temp File System\n"); + reset(); + if (destinationFS.format() && destinationFS.begin()) // must format then mount the new FS + { + if (_copyFS(tempFS, destinationFS)) + { + FSTOOLSDEBUG("Files copied back to new FS\n"); + result = true; + } + } + else + { + FSTOOLSDEBUG("Error Mounting\n"); + } + } + else + { + FSTOOLSDEBUG("Copy Failed\n"); + } + tempFS.end(); + } + else + { + FSTOOLSDEBUG("Failed to begin() TempFS\n"); + } + return result; +}; + +void FSTools::reset() +{ + _mounted = false; + _layout = nullptr; + if (_pFS) + { + _pFS->end(); + _pFS.reset(); + } +} + +void FSTools::fileListIterator(FS & fs, const char * dirName, FST::FileCb Cb) +{ + Dir dir = fs.openDir(dirName); + while (dir.next()) + { + if (dir.isFile()) + { + File f = dir.openFile("r"); + if (Cb) + { + Cb(f); + } + } + else + { + fileListIterator(fs, dir.fileName().c_str(), Cb); + } + } +} + + +uint32_t FSTools::_getStartAddr(const FST::layout & layout) +{ + return (layout.startAddr - 0x40200000); +} + +uint32_t FSTools::_getSize(const FST::layout & layout) +{ + return (layout.endAddr - layout.startAddr); +} + +#ifdef DEBUG_ESP_CORE +void FSTools::_dumpFileInfo(File & f) +{ + if (f) + { + DEBUG_ESP_PORT.printf_P(PSTR(" File: %-30s [%8uB]\n"), f.fullName(), f.size()); + } +} +#endif + +bool FSTools::_copyFS(FS & sourceFS, FS & destFS) +{ + uint32_t sourceFileCount = 0; + uint32_t sourceByteTotal = 0; + + fileListIterator(sourceFS, "/", [&sourceFileCount, &sourceByteTotal](File & f) + { + if (f) + { + sourceFileCount++; + sourceByteTotal += f.size(); + } + }); + + size_t count = 0; + fileListIterator(sourceFS, "/", [&count, &destFS](File & sourceFile) + { + if (sourceFile) + { + File destFile = destFS.open(sourceFile.fullName(), "w"); + if (destFile) + { + destFile.setTimeout(5000); // this value was chosen empirically as it failed with default timeout. + size_t written = destFile.write(sourceFile); + if (written == sourceFile.size()) + { + count++; + } + } + destFile.close(); + sourceFile.close(); + yield(); + } + }); + + return (count == sourceFileCount); + +} diff --git a/libraries/FSTools/FSTools.h b/libraries/FSTools/FSTools.h new file mode 100644 index 000000000..4722a1742 --- /dev/null +++ b/libraries/FSTools/FSTools.h @@ -0,0 +1,91 @@ +#pragma once + +#include +#include +#include +/* + + A temporary FS is made between the END of the sketch... and the start of the partition you try to mount, to maximise the available space for copying the FS. + The WORST case this is at 0x40300000 which is for a 3m FS on 4m flash.. leaving 460Kb for copying. + +*/ + + + +namespace FST +{ + + struct layout + { + constexpr layout(uint32_t s, uint32_t e, uint32_t p, uint32_t b) : startAddr(s), endAddr(e), page(p), block(b) {}; + const uint32_t startAddr{0}; + const uint32_t endAddr{0}; + const uint32_t page{0}; + const uint32_t block{0}; + }; + + enum FS_t : uint8_t + { + SPIFFS, + LITTLEFS + }; + + static constexpr layout layout_512k32 = { 0x40273000, 0x4027B000, 0x100, 0x1000 }; + static constexpr layout layout_512k64 = { 0x4026B000, 0x4027B000, 0x100, 0x1000 }; + static constexpr layout layout_512k128 = { 0x4025B000, 0x4027B000, 0x100, 0x1000 }; + + static constexpr layout layout_1m64 = { 0x402EB000, 0x402FB000, 0x100, 0x1000 }; + static constexpr layout layout_1m128 = { 0x402DB000, 0x402FB000, 0x100, 0x1000 }; + static constexpr layout layout_1m144 = { 0x402D7000, 0x402FB000, 0x100, 0x1000 }; + static constexpr layout layout_1m160 = { 0x402D3000, 0x402FB000, 0x100, 0x1000 }; + static constexpr layout layout_1m192 = { 0x402CB000, 0x402FB000, 0x100, 0x1000 }; + static constexpr layout layout_1m256 = { 0x402BB000, 0x402FB000, 0x100, 0x1000 }; + static constexpr layout layout_1m512 = { 0x4027B000, 0x402FB000, 0x100, 0x2000 }; + + static constexpr layout layout_2m64 = { 0x403F0000, 0x403FB000, 0x100, 0x1000 }; + static constexpr layout layout_2m128 = { 0x403E0000, 0x403FB000, 0x100, 0x1000 }; + static constexpr layout layout_2m256 = { 0x403C0000, 0x403FB000, 0x100, 0x1000 }; + static constexpr layout layout_2m512 = { 0x40380000, 0x403FA000, 0x100, 0x2000 }; + static constexpr layout layout_2m1m = { 0x40300000, 0x403FA000, 0x100, 0x2000 }; + + static constexpr layout layout_4m1m = { 0x40500000, 0x405FA000, 0x100, 0x2000 }; + static constexpr layout layout_4m2m = { 0x40400000, 0x405FA000, 0x100, 0x2000 }; + static constexpr layout layout_4m3m = { 0x40300000, 0x405FA000, 0x100, 0x2000 }; + + static constexpr layout layout_8m6m = { 0x40400000, 0x409FA000, 0x100, 0x2000 }; + static constexpr layout layout_8m7m = { 0x40300000, 0x409FA000, 0x100, 0x2000 }; + + static constexpr layout layout_16m14m = { 0x40400000, 0x411FA000, 0x100, 0x2000 }; + static constexpr layout layout_16m15m = { 0x40300000, 0x411FA000, 0x100, 0x2000 }; + + typedef std::function FileCb; + +}; + + +class FSTools +{ +public: + + FSTools(); + ~FSTools(); + bool attemptToMountFS(fs::FS & fs); + bool mountAlternativeFS(FST::FS_t type, const FST::layout & layout, bool keepMounted = false); + bool mounted(); + bool moveFS(fs::FS & destinationFS); + void reset(); + void fileListIterator(FS & fs, const char * dirName, FST::FileCb Cb); + +private: + uint32_t _getStartAddr(const FST::layout & layout); + uint32_t _getSize(const FST::layout & layout); +#ifdef DEBUG_ESP_CORE + void _dumpFileInfo(File & f); +#endif + bool _copyFS(FS & sourceFS, FS & destFS); + + std::unique_ptr _pFS; + bool _mounted{false}; + const FST::layout * _layout{nullptr}; + +}; \ No newline at end of file diff --git a/libraries/FSTools/examples/Basic_example/Basic_example.ino b/libraries/FSTools/examples/Basic_example/Basic_example.ino new file mode 100644 index 000000000..efa8b5740 --- /dev/null +++ b/libraries/FSTools/examples/Basic_example/Basic_example.ino @@ -0,0 +1,130 @@ +/* + + This sketch will convert SPIFFS partitions to LittleFS on ESP8266 + + Change the `TARGET_FS_LAYOUT` to the partition layout that you want target + ie what you are trying to copy from. + + Include in the sketch whatever you want the destination to be, in this case LittleFS, + but it could be SPIFFS to convert back if need be. + + How it works: It creates a LittleFS partition between the end of the sketch and the + start of whatever filesystem you set as target. This has IMPORTANT implications for the + amount of data you can move!!! eg a 4Mb flash module with a 3Mb SPIFFS partition only leaves + about 450k for the temp file system, so if you have more data than that on your 3Mb SPIFFS it + will fail. + +*/ + + + +#include +#include +#include +#include +#include +#include + + +#define TARGET_FS_LAYOUT FST::layout_4m3m + +FSTools fstools; + +#ifndef STASSID +#define STASSID "xxxx" +#define STAPSK "xxxx" +#endif + +const char* ssid = STASSID; +const char* password = STAPSK; + + +bool migrateFS() { + if (!fstools.attemptToMountFS(LittleFS)) { // Attempts to mount LittleFS without autoformat... + Serial.println(F("Default FS not found")); + if (fstools.mountAlternativeFS(FST::SPIFFS /* FST::LITTLEFS */, TARGET_FS_LAYOUT, true)) { + Serial.println(F("Alternative found")); + if (fstools.moveFS(LittleFS)) { + Serial.println(F("FileSystem Moved New FS contents:")); + fstools.fileListIterator(LittleFS, "/", [](File& f) { + Serial.printf_P(PSTR(" File: %-30s [%8uB]\n"), f.fullName(), f.size()); + }); + return true; + } + } + } + return false; +} + + +void initWiFiOTA() { + WiFi.mode(WIFI_STA); + WiFi.begin(ssid, password); + while (WiFi.waitForConnectResult() != WL_CONNECTED) { + Serial.println("Connection Failed! Rebooting..."); + delay(5000); + ESP.restart(); + } + + ArduinoOTA.onStart([]() { + String type; + if (ArduinoOTA.getCommand() == U_FLASH) { + type = "sketch"; + } else { // U_FS + type = "filesystem"; + } + + // NOTE: if updating FS this would be the place to unmount FS using FS.end() + Serial.println("Start updating " + type); + }); + ArduinoOTA.onEnd([]() { + Serial.println("\nEnd"); + }); + ArduinoOTA.onProgress([](unsigned int progress, unsigned int total) { + Serial.printf("Progress: %u%%\r", (progress / (total / 100))); + }); + ArduinoOTA.onError([](ota_error_t error) { + Serial.printf("Error[%u]: ", error); + if (error == OTA_AUTH_ERROR) { + Serial.println("Auth Failed"); + } else if (error == OTA_BEGIN_ERROR) { + Serial.println("Begin Failed"); + } else if (error == OTA_CONNECT_ERROR) { + Serial.println("Connect Failed"); + } else if (error == OTA_RECEIVE_ERROR) { + Serial.println("Receive Failed"); + } else if (error == OTA_END_ERROR) { + Serial.println("End Failed"); + } + }); + ArduinoOTA.begin(); +} + +void setup() { + + WiFi.persistent(false); + WiFi.disconnect(true); + Serial.begin(115200); + + Serial.println(); + Serial.printf("SDK Version: %s\n", ESP.getSdkVersion()); + Serial.println("Core Version: " + ESP.getCoreVersion()); + Serial.println("Full Version: " + ESP.getFullVersion()); + + Serial.printf("Sketch size: %u\n", ESP.getSketchSize()); + Serial.printf("Free size: %u\n", ESP.getFreeSketchSpace()); + + Serial.println("Booting"); + + migrateFS(); // MUST call this before calling your own begin(); + + initWiFiOTA(); + + Serial.println("Ready"); + Serial.print("IP address: "); + Serial.println(WiFi.localIP()); +} + +void loop() { + ArduinoOTA.handle(); +} diff --git a/libraries/FSTools/examples/custom_FS_example/custom_FS_example.ino b/libraries/FSTools/examples/custom_FS_example/custom_FS_example.ino new file mode 100644 index 000000000..3dc2e804f --- /dev/null +++ b/libraries/FSTools/examples/custom_FS_example/custom_FS_example.ino @@ -0,0 +1,137 @@ +/* + + This sketch will convert SPIFFS partitions to a custom FS on ESP8266 + + Change the `TARGET_FS_LAYOUT` to the partition layout that you want target + ie what you are trying to copy from. + + This ksetch shows how to create a FS different to the one provided for by the sketch defaults. + This is useful if you need to use an intermediate sketch to move the FS but need to maintain the + sketch size limit of say 512kb. + + How it works: It creates a LittleFS partition between the end of the sketch and the + start of whatever filesystem you set as target. This has IMPORTANT implications for the + amount of data you can move!!! eg a 4Mb flash module with a 3Mb SPIFFS partition only leaves + about 450k for the temp file system, so if you have more data than that on your 3Mb SPIFFS it + will fail. + +*/ + + + +#include +#include +#include +#include +#include +#include + + +#define TARGET_FS_LAYOUT FST::layout_4m3m + +const uint32_t startSector = FST::layout_4m1m.startAddr - 0x40200000; +const uint32_t tempFSsize = FST::layout_4m1m.endAddr - FST::layout_4m1m.startAddr; + +fs::FS LittleFS_Different = FS(FSImplPtr(new littlefs_impl::LittleFSImpl(startSector, tempFSsize, FS_PHYS_PAGE, FS_PHYS_BLOCK, 5))); + + +FSTools fstools; + +#ifndef STASSID +#define STASSID "xxxx" +#define STAPSK "xxxx" +#endif + +const char* ssid = STASSID; +const char* password = STAPSK; + + +bool migrateFS() { + if (!fstools.attemptToMountFS(LittleFS_Different)) { // Attempts to mount LittleFS without autoformat... + Serial.println(F("Default FS not found")); + if (fstools.mountAlternativeFS(FST::SPIFFS /* FST::LITTLEFS */, TARGET_FS_LAYOUT, true)) { + Serial.println(F("Alternative found")); + if (fstools.moveFS(LittleFS_Different)) { + Serial.println(F("FileSystem Moved New FS contents:")); + fstools.fileListIterator(LittleFS_Different, "/", [](File& f) { + Serial.printf_P(PSTR(" File: %-30s [%8uB]\n"), f.fullName(), f.size()); + }); + return true; + } + } + } + return false; +} + + +void initWiFiOTA() { + WiFi.mode(WIFI_STA); + WiFi.begin(ssid, password); + while (WiFi.waitForConnectResult() != WL_CONNECTED) { + Serial.println("Connection Failed! Rebooting..."); + delay(5000); + ESP.restart(); + } + + ArduinoOTA.onStart([]() { + String type; + if (ArduinoOTA.getCommand() == U_FLASH) { + type = "sketch"; + } else { // U_FS + type = "filesystem"; + } + + // NOTE: if updating FS this would be the place to unmount FS using FS.end() + Serial.println("Start updating " + type); + }); + ArduinoOTA.onEnd([]() { + Serial.println("\nEnd"); + }); + ArduinoOTA.onProgress([](unsigned int progress, unsigned int total) { + Serial.printf("Progress: %u%%\r", (progress / (total / 100))); + }); + ArduinoOTA.onError([](ota_error_t error) { + Serial.printf("Error[%u]: ", error); + if (error == OTA_AUTH_ERROR) { + Serial.println("Auth Failed"); + } else if (error == OTA_BEGIN_ERROR) { + Serial.println("Begin Failed"); + } else if (error == OTA_CONNECT_ERROR) { + Serial.println("Connect Failed"); + } else if (error == OTA_RECEIVE_ERROR) { + Serial.println("Receive Failed"); + } else if (error == OTA_END_ERROR) { + Serial.println("End Failed"); + } + }); + ArduinoOTA.begin(); +} + +void setup() { + + WiFi.persistent(false); + WiFi.disconnect(true); + Serial.begin(115200); + + Serial.println(); + Serial.printf("SDK Version: %s\n", ESP.getSdkVersion()); + Serial.println("Core Version: " + ESP.getCoreVersion()); + Serial.println("Full Version: " + ESP.getFullVersion()); + + Serial.printf("Sketch size: %u\n", ESP.getSketchSize()); + Serial.printf("Free size: %u\n", ESP.getFreeSketchSpace()); + + Serial.println("Booting"); + + migrateFS(); // MUST call this before calling your own begin(); + + initWiFiOTA(); + + Serial.println("Ready"); + Serial.print("IP address: "); + Serial.println(WiFi.localIP()); +} + +void loop() { + ArduinoOTA.handle(); +} diff --git a/libraries/FSTools/package.json b/libraries/FSTools/package.json new file mode 100644 index 000000000..d10f43fc8 --- /dev/null +++ b/libraries/FSTools/package.json @@ -0,0 +1,10 @@ +{ + "name": "FSTools", + "keywords": "SPIFFS LittleFS", + "description": "A library that manages convertion between SPIFFS and LITTLEFS as well as mounting partitions outside of sketch default.", + "homepage": "", + "author": "sticilface", + "version": "1.0.0", + "frameworks": "arduino", + "platforms": "esp8266" + } \ No newline at end of file