Merge branch 'esp8266' of https://github.com/ficeto/Arduino into ficeto-esp8266

cores/esp8266/Arduino.h

@@ -38,6 +38,7 @@ extern "C" {
 #include "pgmspace.h"
 #include "esp8266_peri.h"
 #include "twi.h"
+#include "spiffs/spiffs.h"
 
 void yield(void);
 
@@ -211,6 +212,7 @@ void loop(void);
 #include "WString.h"
 
 #include "HardwareSerial.h"
+#include "FileSystem.h"
 #include "Esp.h"
 
 uint16_t makeWord(uint16_t w);

cores/esp8266/FileSystem.cpp

@@ -0,0 +1,178 @@
+/****
+ * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development.
+ * Created 2015 by Skurydin Alexey
+ * http://github.com/anakod/Sming
+ * All files of the Sming Core are provided under the LGPL v3 license.
+ ****/
+
+#include "FileSystem.h"
+#include "WString.h"
+
+file_t fileOpen(const String name, FileOpenFlags flags)
+{
+  int repeats = 0;
+  bool notExist;
+  bool canRecreate = (flags & eFO_CreateIfNotExist) == eFO_CreateIfNotExist;
+  int res;
+
+  do
+  {
+	  notExist = false;
+	  res = SPIFFS_open(&_filesystemStorageHandle, name.c_str(), (spiffs_flags)flags, 0);
+	  int code = SPIFFS_errno(&_filesystemStorageHandle);
+	  if (res < 0)
+	  {
+		  debugf("open errno %d\n", code);
+		  notExist = (code == SPIFFS_ERR_NOT_FOUND || code == SPIFFS_ERR_DELETED || code == SPIFFS_ERR_FILE_DELETED || code == SPIFFS_ERR_IS_FREE);
+		  //debugf("recreate? %d %d %d", notExist, canRecreate, (repeats < 3));
+		  if (notExist && canRecreate)
+			  fileDelete(name); // fix for deleted files
+	  }
+  } while (notExist && canRecreate && repeats++ < 3);
+
+  return res;
+}
+
+void fileClose(file_t file)
+{
+  SPIFFS_close(&_filesystemStorageHandle, file);
+}
+
+size_t fileWrite(file_t file, const void* data, size_t size)
+{
+  int res = SPIFFS_write(&_filesystemStorageHandle, file, (void *)data, size);
+  if (res < 0)
+  {
+    debugf("write errno %d\n", SPIFFS_errno(&_filesystemStorageHandle));
+    return res;
+  }
+  return res;
+}
+
+size_t fileRead(file_t file, void* data, size_t size)
+{
+  int res = SPIFFS_read(&_filesystemStorageHandle, file, data, size);
+  if (res < 0)
+  {
+    debugf("read errno %d\n", SPIFFS_errno(&_filesystemStorageHandle));
+    return res;
+  }
+  return res;
+}
+
+int fileSeek(file_t file, int offset, SeekOriginFlags origin)
+{
+  return SPIFFS_lseek(&_filesystemStorageHandle, file, offset, origin);
+}
+
+bool fileIsEOF(file_t file)
+{
+  return SPIFFS_eof(&_filesystemStorageHandle, file);
+}
+
+int32_t fileTell(file_t file)
+{
+  return SPIFFS_tell(&_filesystemStorageHandle, file);
+}
+
+int fileFlush(file_t file)
+{
+  return SPIFFS_fflush(&_filesystemStorageHandle, file);
+}
+
+int fileStats(const String name, spiffs_stat *stat)
+{
+	return SPIFFS_stat(&_filesystemStorageHandle, name.c_str(), stat);
+}
+
+int fileStats(file_t file, spiffs_stat *stat)
+{
+	return SPIFFS_fstat(&_filesystemStorageHandle, file, stat);
+}
+
+void fileDelete(const String name)
+{
+	SPIFFS_remove(&_filesystemStorageHandle, name.c_str());
+}
+
+void fileDelete(file_t file)
+{
+	SPIFFS_fremove(&_filesystemStorageHandle, file);
+}
+
+bool fileExist(const String name)
+{
+  spiffs_stat stat = {0};
+  if (fileStats(name.c_str(), &stat) < 0) return false;
+  return stat.name[0] != '\0';
+}
+
+
+int fileLastError(file_t fd)
+{
+  return SPIFFS_errno(&_filesystemStorageHandle);
+}
+
+void fileClearLastError(file_t fd)
+{
+  _filesystemStorageHandle.errno = SPIFFS_OK;
+}
+
+void fileSetContent(const String fileName, const char *content)
+{
+	file_t file = fileOpen(fileName.c_str(), eFO_CreateNewAlways | eFO_WriteOnly);
+	fileWrite(file, content, os_strlen(content));
+	fileClose(file);
+}
+
+uint32_t fileGetSize(const String fileName)
+{
+	file_t file = fileOpen(fileName.c_str(), eFO_ReadOnly);
+	// Get size
+	fileSeek(file, 0, eSO_FileEnd);
+	int size = fileTell(file);
+	fileClose(file);
+	return size;
+}
+
+String fileGetContent(const String fileName)
+{
+	file_t file = fileOpen(fileName.c_str(), eFO_ReadOnly);
+	// Get size
+	fileSeek(file, 0, eSO_FileEnd);
+	int size = fileTell(file);
+	if (size <= 0)
+	{
+		fileClose(file);
+		return "";
+	}
+	fileSeek(file, 0, eSO_FileStart);
+	char* buffer = new char[size + 1];
+	buffer[size] = 0;
+	fileRead(file, buffer, size);
+	fileClose(file);
+	String res = buffer;
+	delete[] buffer;
+	return res;
+}
+
+int fileGetContent(const String fileName, char* buffer, int bufSize)
+{
+	if (buffer == NULL || bufSize == 0) return 0;
+	*buffer = 0;
+
+	file_t file = fileOpen(fileName.c_str(), eFO_ReadOnly);
+	// Get size
+	fileSeek(file, 0, eSO_FileEnd);
+	int size = fileTell(file);
+	if (size <= 0 || bufSize <= size)
+	{
+		fileClose(file);
+		return 0;
+	}
+	buffer[size] = 0;
+	fileSeek(file, 0, eSO_FileStart);
+	fileRead(file, buffer, size);
+	fileClose(file);
+	return size;
+}

cores/esp8266/FileSystem.h

@@ -0,0 +1,59 @@
+/****
+ * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development.
+ * Created 2015 by Skurydin Alexey
+ * http://github.com/anakod/Sming
+ * All files of the Sming Core are provided under the LGPL v3 license.
+ ****/
+
+#ifndef _SMING_CORE_FILESYSTEM_H_
+#define _SMING_CORE_FILESYSTEM_H_
+
+#include "spiffs/spiffs.h"
+class String;
+
+enum FileOpenFlags
+{
+  eFO_ReadOnly = SPIFFS_RDONLY,
+  eFO_WriteOnly = SPIFFS_WRONLY,
+  eFO_ReadWrite = eFO_ReadOnly | eFO_WriteOnly,
+  eFO_CreateIfNotExist = SPIFFS_CREAT,
+  eFO_Append = SPIFFS_APPEND,
+  eFO_Truncate = SPIFFS_TRUNC,
+  eFO_CreateNewAlways = eFO_CreateIfNotExist | eFO_Truncate
+};
+
+static FileOpenFlags operator|(FileOpenFlags lhs, FileOpenFlags rhs)
+{
+    return (FileOpenFlags) ((int)lhs| (int)rhs);
+}
+
+typedef enum
+{
+	eSO_FileStart = SPIFFS_SEEK_SET,
+	eSO_CurrentPos = SPIFFS_SEEK_CUR,
+	eSO_FileEnd = SPIFFS_SEEK_END
+} SeekOriginFlags;
+
+file_t fileOpen(const String name, FileOpenFlags flags);
+void fileClose(file_t file);
+size_t fileWrite(file_t file, const void* data, size_t size);
+size_t fileRead(file_t file, void* data, size_t size);
+int fileSeek(file_t file, int offset, SeekOriginFlags origin);
+bool fileIsEOF(file_t file);
+int32_t fileTell(file_t file);
+int fileFlush(file_t file);
+int fileLastError(file_t fd);
+void fileClearLastError(file_t fd);
+void fileSetContent(const String fileName, const char *content);
+uint32_t fileGetSize(const String fileName);
+String fileGetContent(const String fileName);
+int fileGetContent(const String fileName, char* buffer, int bufSize);
+
+
+int fileStats(const String name, spiffs_stat *stat);
+int fileStats(file_t file, spiffs_stat *stat);
+void fileDelete(const String name);
+void fileDelete(file_t file);
+bool fileExist(const String name);
+
+#endif /* _SMING_CORE_FILESYSTEM_H_ */

cores/esp8266/core_esp8266_wiring.c

@@ -76,6 +76,7 @@ void delayMicroseconds(unsigned int us) {
 void init() {
     initPins();
     timer1_isr_init();
+    //spiffs_mount();
     os_timer_setfn(&micros_overflow_timer, (os_timer_func_t*) &micros_overflow_tick, 0);
     os_timer_arm(&micros_overflow_timer, 60000, REPEAT);
 }

cores/esp8266/spiffs/LICENSE

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2013-2015 Peter Andersson (pelleplutt1976<at>gmail.com)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

cores/esp8266/spiffs/Makefile

@@ -0,0 +1,44 @@
+
+#############################################################
+# Required variables for each makefile
+# Discard this section from all parent makefiles
+# Expected variables (with automatic defaults):
+#   CSRCS (all "C" files in the dir)
+#   SUBDIRS (all subdirs with a Makefile)
+#   GEN_LIBS - list of libs to be generated ()
+#   GEN_IMAGES - list of images to be generated ()
+#   COMPONENTS_xxx - a list of libs/objs in the form
+#     subdir/lib to be extracted and rolled up into
+#     a generated lib/image xxx.a ()
+#
+ifndef PDIR
+GEN_LIBS = spiffs.a
+endif
+
+#############################################################
+# Configuration i.e. compile options etc.
+# Target specific stuff (defines etc.) goes in here!
+# Generally values applying to a tree are captured in the
+#   makefile at its root level - these are then overridden
+#   for a subtree within the makefile rooted therein
+#
+#DEFINES += 
+
+#############################################################
+# Recursion Magic - Don't touch this!!
+#
+# Each subtree potentially has an include directory
+#   corresponding to the common APIs applicable to modules
+#   rooted at that subtree. Accordingly, the INCLUDE PATH
+#   of a module can only contain the include directories up
+#   its parent path, and not its siblings
+#
+# Required for each makefile to inherit from the parent
+#
+
+INCLUDES := $(INCLUDES) -I $(PDIR)include
+INCLUDES += -I ./
+INCLUDES += -I ../libc
+INCLUDES += -I ../platform
+PDIR := ../$(PDIR)
+sinclude $(PDIR)Makefile

cores/esp8266/spiffs/docs/INTEGRATION

@@ -0,0 +1,306 @@
+* QUICK AND DIRTY INTEGRATION EXAMPLE
+
+So, assume you're running a Cortex-M3 board with a 2 MB SPI flash on it. The 
+SPI flash has 64kB blocks. Your project is built using gnumake, and now you 
+want to try things out.
+
+First, you simply copy the files in src/ to your own source folder. Exclude
+all files in test folder. Then you point out these files in your make script 
+for compilation.
+
+Also copy the spiffs_config.h over from the src/default/ folder.
+
+Try building. This fails, nagging about inclusions and u32_t and whatnot. Open 
+the spiffs_config.h and delete the bad inclusions. Also, add following 
+typedefs:
+
+  typedef signed int s32_t;
+  typedef unsigned int u32_t;
+  typedef signed short s16_t;
+  typedef unsigned short u16_t;
+  typedef signed char s8_t;
+  typedef unsigned char u8_t;
+  
+Now it should build. Over to the mounting business. Assume you already 
+implemented the read, write and erase functions to your SPI flash:
+
+  void my_spi_read(int addr, int size, char *buf)
+  void my_spi_write(int addr, int size, char *buf)
+  void my_spi_erase(int addr, int size)
+
+In your main.c or similar, include the spiffs.h and do that spiffs struct:
+
+  #include <spiffs.h>
+  
+  static spiffs fs;
+
+Also, toss up some of the needed buffers:
+
+  #define LOG_PAGE_SIZE       256
+  
+  static u8_t spiffs_work_buf[LOG_PAGE_SIZE*2];
+  static u8_t spiffs_fds[32*4];
+  static u8_t spiffs_cache_buf[(LOG_PAGE_SIZE+32)*4];
+  
+Now, write the my_spiffs_mount function:
+
+  void my_spiffs_mount() {
+    spiffs_config cfg;
+    cfg.phys_size = 2*1024*1024; // use all spi flash
+    cfg.phys_addr = 0; // start spiffs at start of spi flash
+    cfg.phys_erase_block = 65536; // according to datasheet
+    cfg.log_block_size = 65536; // let us not complicate things
+    cfg.log_page_size = LOG_PAGE_SIZE; // as we said
+    
+    cfg.hal_read_f = my_spi_read;
+    cfg.hal_write_f = my_spi_write;
+    cfg.hal_erase_f = my_spi_erase;
+    
+    int res = SPIFFS_mount(&fs,
+      &cfg,
+      spiffs_work_buf,
+      spiffs_fds,
+      sizeof(spiffs_fds),
+      spiffs_cache_buf,
+      sizeof(spiffs_cache_buf),
+      0);
+    printf("mount res: %i\n", res);
+  }
+
+Now, build warns about the my_spi_read, write and erase functions. Wrong 
+signatures, so go wrap them:
+
+  static s32_t my_spiffs_read(u32_t addr, u32_t size, u8_t *dst) {
+    my_spi_read(addr, size, dst);
+    return SPIFFS_OK;
+  }
+
+  static s32_t my_spiffs_write(u32_t addr, u32_t size, u8_t *src) {
+    my_spi_write(addr, size, dst);
+    return SPIFFS_OK;
+  }
+
+  static s32_t my_spiffs_erase(u32_t addr, u32_t size) {
+    my_spi_erase(addr, size);
+    return SPIFFS_OK;
+  } 
+
+Redirect the config in my_spiffs_mount to the wrappers instead:
+
+    cfg.hal_read_f = my_spiffs_read;
+    cfg.hal_write_f = my_spiffs_write;
+    cfg.hal_erase_f = my_spiffs_erase;
+
+Ok, now you should be able to build and run. However, you get this output:
+  
+  mount res: -1
+  
+but you wanted 
+
+  mount res: 0
+  
+This is probably due to you having experimented with your SPI flash, so it
+contains rubbish from spiffs's point of view. Do a mass erase and run again.
+
+If all is ok now, you're good to go. Try creating a file and read it back:
+
+  static void test_spiffs() {
+    char buf[12];
+    
+    // Surely, I've mounted spiffs before entering here
+  
+    spiffs_file fd = SPIFFS_open(&fs, "my_file", SPIFFS_CREAT | SPIFFS_TRUNC | SPIFFS_RDWR, 0);
+    if (SPIFFS_write(&fs, fd, (u8_t *)"Hello world", 12) < 0) printf("errno %i\n", SPIFFS_errno(&fs));
+    SPIFFS_close(&fs, fd); 
+  
+    fd = SPIFFS_open(&fs, "my_file", SPIFFS_RDWR, 0);
+    if (SPIFFS_read(&fs, fd, (u8_t *)buf, 12) < 0) printf("errno %i\n", SPIFFS_errno(&fs));
+    SPIFFS_close(&fs, fd);
+  
+    printf("--> %s <--\n", buf);
+  }
+  
+Compile, run, cross fingers hard, and you'll get the output:
+
+  --> Hello world <-- 
+  
+Got errors? Check spiffs.h for error definitions to get a clue what went voodoo.
+
+
+* THINGS TO CHECK
+
+When you alter the spiffs_config values, make sure you also check the typedefs
+in spiffs_config.h:
+
+ - spiffs_block_ix
+ - spiffs_page_ix
+ - spiffs_obj_id
+ - spiffs_span_ix
+
+The sizes of these typedefs must not underflow, else spiffs might end up in
+eternal loops. Each typedef is commented what check for.
+
+Also, if you alter the code or just want to verify your configuration, you can
+run
+
+ > make test
+
+in the spiffs folder. This will run all testcases using the configuration in
+default/spiffs_config.h and test/params_test.h. The tests are written for linux
+but should run under cygwin also.
+
+
+* INTEGRATING SPIFFS
+
+In order to integrate spiffs to your embedded target, you will basically need:
+ - A SPI flash device which your processor can communicate with
+ - An implementation for reading, writing and erasing the flash
+ - Memory (flash or ram) for the code
+ - Memory (ram) for the stack
+
+Other stuff may be needed, threaded systems might need mutexes and so on. 
+
+** Logical structure
+
+First and foremost, one must decide how to divide up the SPI flash for spiffs. 
+Having the datasheet for the actual SPI flash in hand will help. Spiffs can be 
+defined to use all or only parts of the SPI flash.
+
+If following seems arcane, read the "HOW TO CONFIG" chapter first.
+
+ - Decide the logical size of blocks. This must be a multiple of the biggest 
+   physical SPI flash block size. To go safe, use the physical block size - 
+   which in many cases is 65536 bytes.
+ - Decide the logical size of pages. This must be a 2nd logarithm part of the
+   logical block size. To go safe, use 256 bytes to start with.
+ - Decide how much of the SPI flash memory to be used for spiffs. This must be
+   on logical block boundary. If unsafe, use 1 megabyte to start with.
+ - Decide where on the SPI flash memory the spiffs area should start. This must
+   be on physical block/sector boundary. If unsafe, use address 0.
+
+** SPI flash API
+
+The target must provide three functions to spiffs:
+
+ - s32_t (*spiffs_read)(u32_t addr, u32_t size, u8_t *dst)
+ - s32_t (*spiffs_write)(u32_t addr, u32_t size, u8_t *src)
+ - s32_t (*spiffs_erase)(u32_t addr, u32_t size)
+ 
+These functions define the only communication between the SPI flash and the
+spiffs stack. 
+ 
+On success these must return 0 (or SPIFFS_OK). Anything else will be considered
+an error. 
+
+The size for read and write requests will never exceed the logical page size,
+but it may be less.
+
+The address and size on erase requests will always be on physical block size
+boundaries.  
+
+** Mount specification
+
+In spiffs.h, there is a SPIFFS_mount function defined, used to mount spiffs on
+the SPI flash.
+
+s32_t SPIFFS_mount(
+	spiffs *fs, 
+	spiffs_config *config, 
+	u8_t *work,
+    u8_t *fd_space, 
+    u32_t fd_space_size,
+    void *cache, 
+    u32_t cache_size,
+    spiffs_check_callback check_cb_f)
+    
+ - fs            Points to a spiffs struct. This may be totally uninitialized.
+ - config        Points to a spiffs_config struct. This struct must be
+                 initialized when mounting. See below.
+ - work          A ram memory buffer being double the size of the logical page
+                 size. This buffer is used excessively by the spiffs stack. If
+                 logical page size is 256, this buffer must be 512 bytes.
+ - fd_space      A ram memory buffer used for file descriptors.
+ - fd_space_size The size of the file descriptor buffer. A file descriptor
+                 normally is around 32 bytes depending on the build config -
+                 the bigger the buffer, the more file descriptors are
+                 available.
+ - cache         A ram memory buffer used for cache. Ignored if cache is
+                 disabled in build config. 
+ - cache_size    The size of the cache buffer. Ignored if cache is disabled in
+                 build config. One cache page will be slightly larger than the 
+                 logical page size. The more ram, the more cache pages, the 
+                 quicker the system.
+ - check_cb_f    Callback function for monitoring spiffs consistency checks and
+                 mending operations. May be null.
+
+The config struct must be initialized prior to mounting. One must always 
+define the SPI flash access functions:
+
+ spiffs_config.hal_read_f - pointing to the function reading the SPI flash
+
+ spiffs_config.hal_write_f - pointing to the function writing the SPI flash
+
+ spiffs_config.hal_erase_f - pointing to the function erasing the SPI flash
+ 
+Depending on the build config - if SPIFFS_SINGLETON is set to zero - following
+parameters must be defined:
+ 
+ spiffs_config.phys_size - the physical number of bytes accounted for 
+                           spiffs on the SPI flash
+
+ spiffs_config.phys_addr - the physical starting address on the SPI flash
+
+ spiffs_config.phys_erase_block - the physical size of the largest block/sector
+                                  on the SPI flash found within the spiffs
+                                  usage address space
+
+ spiffs_config.log_block_size - the logical size of a spiffs block
+
+ spiffs_config.log_page_size - the logical size of a spiffs page
+ 
+If SPIFFS_SINGLETON is set to one, above parameters must be set ny defines in
+the config header file, spiffs_config.h.
+
+
+** Build config
+
+makefile: The files needed to be compiled to your target resides in files.mk to
+be included in your makefile, either by cut and paste or by inclusion.
+
+Types: spiffs uses the types u8_t, s8_t, u16_t, s16_t, u32_t, s32_t; these must 
+be typedeffed.
+
+spiffs_config.h: you also need to define a spiffs_config.h header. Example of
+this is found in the default/ directory.
+
+
+** RAM
+
+Spiffs needs ram. It needs a working buffer being double the size of the 
+logical page size. It also needs at least one file descriptor. If cache is 
+enabled (highly recommended), it will also need a bunch of cache pages.
+
+Say you have a logical page size of 256 bytes. You want to be able to have four
+files open simultaneously, and you can give spiffs four cache pages. This 
+roughly sums up to:
+
+256*2 (work buffer) + 
+32*4 (file descriptors) + 
+(256+32)*4 (cache pages) + 40 (cache metadata)
+
+i.e. 1832 bytes. 
+
+This is apart from call stack usage.
+
+To get the exact amount of bytes needed on your specific target, enable 
+SPIFFS_BUFFER_HELP in spiffs_config.h, rebuild and call:
+
+  SPIFFS_buffer_bytes_for_filedescs
+  SPIFFS_buffer_bytes_for_cache
+  
+Having these figures you can disable SPIFFS_BUFFER_HELP again to save flash.
+ 
+
+* HOW TO CONFIG
+
+TODO

cores/esp8266/spiffs/docs/TECH_SPEC

@@ -0,0 +1,239 @@
+* USING SPIFFS
+
+TODO
+
+
+* SPIFFS DESIGN
+
+Spiffs is inspired by YAFFS. However, YAFFS is designed for NAND flashes, and
+for bigger targets with much more ram. Nevertheless, many wise thoughts have
+been borrowed from YAFFS when writing spiffs. Kudos!
+
+The main complication writing spiffs was that it cannot be assumed the target
+has a heap. Spiffs must go along only with the work ram buffer given to it. 
+This forces extra implementation on many areas of spiffs.
+
+
+** SPI flash devices using NOR technology
+
+Below is a small description of how SPI flashes work internally. This is to
+give an understanding of the design choices made in spiffs.
+
+SPI flash devices are physically divided in blocks. On some SPI flash devices,
+blocks are further divided into sectors. Datasheets sometimes name blocks as 
+sectors and vice versa.
+
+Common memory capacaties for SPI flashes are 512kB up to 8MB of data, where
+blocks may be 64kB. Sectors can be e.g. 4kB, if supported. Many SPI flashes 
+have uniform block sizes, whereas others have non-uniform - the latter meaning 
+that e.g. the first 16 blocks are 4kB big, and the rest are 64kB.
+
+The entire memory is linear and can be read and written in random access. 
+Erasing can only be done block- or sectorwise; or by mass erase.
+
+SPI flashes can normally be erased from 100.000 up to 1.000.000 cycles before
+they fail.
+
+A clean SPI flash from factory have all bits in entire memory set to one. A
+mass erase will reset the device to this state. Block or sector erasing will
+put the all bits in the area given by the sector or block to ones. Writing to a
+NOR flash pulls ones to zeroes. Writing 0xFF to an address is simply a no-op. 
+
+Writing 0b10101010 to a flash address holding 0b00001111 will yield 0b00001010.
+
+This way of "write by nand" is used considerably in spiffs.
+
+Common characteristics of NOR flashes are quick reads, but slow writes.
+
+And finally, unlike NAND flashes, NOR flashes seem to not need any error 
+correction. They always write correctly I gather.
+
+
+** Spiffs logical structure
+
+Some terminology before proceeding. Physical blocks/sectors means sizes stated
+in the datasheet. Logical blocks and pages is something the integrator choose.
+
+
+** Blocks and pages
+
+Spiffs is allocated to a part or all of the memory of the SPI flash device. 
+This area is divided into logical blocks, which in turn are divided into 
+logical pages. The boundary of a logical block must coincide with one or more 
+physical blocks. The sizes for logical blocks and logical pages always remain
+the same, they are uniform.
+
+Example: non-uniform flash mapped to spiffs with 128kB logical blocks
+
+PHYSICAL FLASH BLOCKS               SPIFFS LOGICAL BLOCKS: 128kB
+
++-----------------------+   - - -   +-----------------------+
+| Block 1 : 16kB        |           | Block 1 : 128kB       |
++-----------------------+           |                       |
+| Block 2 : 16kB        |           |                       |
++-----------------------+           |                       |
+| Block 3 : 16kB        |           |                       |
++-----------------------+           |                       |
+| Block 4 : 16kB        |           |                       |
++-----------------------+           |                       |
+| Block 5 : 64kB        |           |                       |
++-----------------------+   - - -   +-----------------------+
+| Block 6 : 64kB        |           | Block 2 : 128kB       |
++-----------------------+           |                       |
+| Block 7 : 64kB        |           |                       |
++-----------------------+   - - -   +-----------------------+
+| Block 8 : 64kB        |           | Block 3 : 128kB       |
++-----------------------+           |                       |
+| Block 9 : 64kB        |           |                       |
++-----------------------+   - - -   +-----------------------+
+| ...                   |           | ...                   |
+
+A logical block is divided further into a number of logical pages. A page 
+defines the smallest data holding element known to spiffs. Hence, if a file
+is created being one byte big, it will occupy one page for index and one page
+for data - it will occupy 2 x size of a logical page on flash.
+So it seems it is good to select a small page size.
+
+Each page has a metadata header being normally 5 to 9 bytes. This said, a very
+small page size will make metadata occupy a lot of the memory on the flash. A
+page size of 64 bytes will waste 8-14% on metadata, while 256 bytes 2-4%.
+So it seems it is good to select a big page size.
+
+Also, spiffs uses a ram buffer being two times the page size. This ram buffer
+is used for loading and manipulating pages, but it is also used for algorithms 
+to find free file ids, scanning the file system, etc. Having too small a page
+size means less work buffer for spiffs, ending up in more reads operations and
+eventually gives a slower file system.
+
+Choosing the page size for the system involves many factors:
+ - How big is the logical block size
+ - What is the normal size of most files
+ - How much ram can be spent
+ - How much data (vs metadata) must be crammed into the file system
+ - How fast must spiffs be
+ - Other things impossible to find out
+ 
+So, chosing the Optimal Page Size (tm) seems tricky, to say the least. Don't 
+fret - there is no optimal page size. This varies from how the target will use
+spiffs. Use the golden rule:
+
+        ~~~   Logical Page Size = Logical Block Size / 256   ~~~
+
+This is a good starting point. The final page size can then be derived through 
+heuristical experimenting for us non-analytical minds.
+
+
+** Objects, indices and look-ups
+
+A file, or an object as called in spiffs, is identified by an object id. 
+Another YAFFS rip-off. This object id is a part of the page header. So, all 
+pages know to which object/file they belong - not counting the free pages.
+
+An object is made up of two types of pages: object index pages and data pages.
+Data pages contain the data written by user. Index pages contain metadata about
+the object, more specifically what data pages are part of the object.
+
+The page header also includes something called a span index. Let's say a file
+is written covering three data pages. The first data page will then have span 
+index 0, the second span index 1, and the last data page will have span index
+2. Simple as that. 
+
+Finally, each page header contain flags, telling if the page is used, 
+deleted, finalized, holds index or data, and more.
+
+Object indices also have span indices, where an object index with span index 0
+is referred to as the object index header. This page does not only contain 
+references to data pages, but also extra info such as object name, object size
+in bytes, flags for file or directory, etc.
+
+If one were to create a file covering three data pages, named e.g. 
+"spandex-joke.txt", given object id 12, it could look like this:
+
+PAGE 0  <things to be unveiled soon>
+
+PAGE 1  page header:   [obj_id:12  span_ix:0  flags:USED|DATA]
+        <first data page of joke>
+
+PAGE 2  page header:   [obj_id:12  span_ix:1  flags:USED|DATA]
+        <second data page of joke>
+
+PAGE 3  page header:   [obj_id:545 span_ix:13 flags:USED|DATA]
+        <some data belonging to object 545, probably not very amusing>
+
+PAGE 4  page header:   [obj_id:12  span_ix:2  flags:USED|DATA]
+        <third data page of joke>
+
+PAGE 5  page header:   [obj_id:12  span_ix:0  flags:USED|INDEX]
+        obj ix header: [name:spandex-joke.txt  size:600 bytes  flags:FILE] 
+        obj ix:        [1 2 4]
+        
+Looking in detail at page 5, the object index header page, the object index
+array refers to each data page in order, as mentioned before. The index of the
+object index array correlates with the data page span index.
+
+                            entry ix:  0 1 2
+                              obj ix: [1 2 4]
+                                       | | |
+    PAGE 1, DATA, SPAN_IX 0    --------/ | |
+      PAGE 2, DATA, SPAN_IX 1    --------/ |
+        PAGE 4, DATA, SPAN_IX 2    --------/
+        
+Things to be unveiled in page 0 - well.. Spiffs is designed for systems low on 
+ram. We cannot keep a dynamic list on the whereabouts of each object index 
+header so we can find a file fast. There might not even be a heap! But, we do 
+not want to scan all page headers on the flash to find the object index header.
+
+The first page(s) of each block contains the so called object look-up. These 
+are not normal pages, they do not have a header. Instead, they are arrays 
+pointing out what object-id the rest of all pages in the block belongs to.
+
+By this look-up, only the first page(s) in each block must to scanned to find 
+the actual page which contains the object index header of the desired object.
+
+The object lookup is redundant metadata. The assumption is that it presents 
+less overhead reading a full page of data to memory from each block and search
+that, instead of reading a small amount of data from each page (i.e. the page 
+header) in all blocks. Each read operation from SPI flash normally contains 
+extra data as the read command itself and the flash address. Also, depending on
+the underlying implementation, other criterions may need to be passed for each 
+read transaction, like mutexes and such.
+
+The veiled example unveiled would look like this, with some extra pages:
+
+PAGE 0  [  12   12  545   12   12   34   34    4    0    0    0    0 ...]
+PAGE 1  page header:   [obj_id:12  span_ix:0  flags:USED|DATA] ...
+PAGE 2  page header:   [obj_id:12  span_ix:1  flags:USED|DATA] ...
+PAGE 3  page header:   [obj_id:545 span_ix:13 flags:USED|DATA] ...
+PAGE 4  page header:   [obj_id:12  span_ix:2  flags:USED|DATA] ...
+PAGE 5  page header:   [obj_id:12  span_ix:0  flags:USED|INDEX] ...
+PAGE 6  page header:   [obj_id:34  span_ix:0  flags:USED|DATA] ...
+PAGE 7  page header:   [obj_id:34  span_ix:1  flags:USED|DATA] ...
+PAGE 8  page header:   [obj_id:4   span_ix:1  flags:USED|INDEX] ...
+PAGE 9  page header:   [obj_id:23  span_ix:0  flags:DELETED|INDEX] ...
+PAGE 10 page header:   [obj_id:23  span_ix:0  flags:DELETED|DATA] ...
+PAGE 11 page header:   [obj_id:23  span_ix:1  flags:DELETED|DATA] ...
+PAGE 12 page header:   [obj_id:23  span_ix:2  flags:DELETED|DATA] ...
+...
+
+Ok, so why are page 9 to 12 marked as 0 when they belong to object id 23? These
+pages are deleted, so this is marked both in page header flags and in the look
+up. This is an example where spiffs uses NOR flashes "nand-way" of writing.
+
+As a matter of fact, there are two object id's which are special:
+
+obj id 0 (all bits zeroes) - indicates a deleted page in object look up  
+obj id 0xff.. (all bits ones) - indicates a free page in object look up 
+
+Actually, the object id's have another quirk: if the most significant bit is
+set, this indicates an object index page. If the most significant bit is zero,
+this indicates a data page. So to be fully correct, page 0 in above example 
+would look like this:
+
+PAGE 0  [  12   12  545   12  *12   34   34   *4    0    0    0    0 ...]
+
+where the asterisk means the msb of the object id is set.
+
+This is another way to speed up the searches when looking for object indices.
+By looking on the object id's msb in the object lookup, it is also possible 
+to find out whether the page is an object index page or a data page.
+

cores/esp8266/spiffs/docs/TODO

@@ -0,0 +1 @@
+* When mending lost pages, also see if they fit into length specified in object index header

cores/esp8266/spiffs/flashmem.h

@@ -0,0 +1,73 @@
+// Based on NodeMCU platform_flash
+// https://github.com/nodemcu/nodemcu-firmware
+
+#ifndef SYSTEM_FLASHMEM_H_
+#define SYSTEM_FLASHMEM_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "spiffs.h"
+#include "spi_flash.h"
+
+#define INTERNAL_FLASH_WRITE_UNIT_SIZE  4
+#define INTERNAL_FLASH_READ_UNIT_SIZE	4
+
+#define FLASH_TOTAL_SEC_COUNT 	(flashmem_get_size_sectors())
+
+#define SYS_PARAM_SEC_COUNT 4
+#define FLASH_WORK_SEC_COUNT (FLASH_TOTAL_SEC_COUNT - SYS_PARAM_SEC_COUNT)
+
+#define INTERNAL_FLASH_SECTOR_SIZE      SPI_FLASH_SEC_SIZE
+#define INTERNAL_FLASH_SIZE             ( (FLASH_WORK_SEC_COUNT) * INTERNAL_FLASH_SECTOR_SIZE )
+#define INTERNAL_FLASH_START_ADDRESS    0x40200000
+
+typedef struct
+{
+    uint8_t unknown0;
+    uint8_t unknown1;
+    enum
+    {
+        MODE_QIO = 0,
+        MODE_QOUT = 1,
+        MODE_DIO = 2,
+        MODE_DOUT = 15,
+    } mode : 8;
+    enum
+    {
+        SPEED_40MHZ = 0,
+        SPEED_26MHZ = 1,
+        SPEED_20MHZ = 2,
+        SPEED_80MHZ = 15,
+    } speed : 4;
+    enum
+    {
+        SIZE_4MBIT = 0,
+        SIZE_2MBIT = 1,
+        SIZE_8MBIT = 2,
+        SIZE_16MBIT = 3,
+        SIZE_32MBIT = 4,
+    } size : 4;
+} STORE_TYPEDEF_ATTR SPIFlashInfo;
+
+extern uint32_t flashmem_write( const void *from, uint32_t toaddr, uint32_t size );
+extern uint32_t flashmem_read( void *to, uint32_t fromaddr, uint32_t size );
+extern bool flashmem_erase_sector( uint32_t sector_id );
+
+extern SPIFlashInfo flashmem_get_info();
+extern uint8_t flashmem_get_size_type();
+extern uint32_t flashmem_get_size_bytes();
+extern uint16_t flashmem_get_size_sectors();
+uint32_t flashmem_find_sector( uint32_t address, uint32_t *pstart, uint32_t *pend );
+uint32_t flashmem_get_sector_of_address( uint32_t addr );
+
+extern uint32_t flashmem_write_internal( const void *from, uint32_t toaddr, uint32_t size );
+extern uint32_t flashmem_read_internal( void *to, uint32_t fromaddr, uint32_t size );
+extern uint32_t flashmem_get_first_free_block_address();
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* SYSTEM_FLASHMEM_H_ */

cores/esp8266/spiffs/spiffs.c

@@ -0,0 +1,181 @@
+#include "spiffs.h"
+
+#define LOG_PAGE_SIZE       256
+
+spiffs _filesystemStorageHandle;
+
+static u8_t spiffs_work_buf[LOG_PAGE_SIZE*2];
+static u8_t spiffs_fds[32*4];
+static u8_t spiffs_cache[(LOG_PAGE_SIZE+32)*4];
+
+static s32_t api_spiffs_read(u32_t addr, u32_t size, u8_t *dst)
+{
+  flashmem_read(dst, addr, size);
+  return SPIFFS_OK;
+}
+
+static s32_t api_spiffs_write(u32_t addr, u32_t size, u8_t *src)
+{
+  //debugf("api_spiffs_write");
+  flashmem_write(src, addr, size);
+  return SPIFFS_OK;
+}
+
+static s32_t api_spiffs_erase(u32_t addr, u32_t size)
+{
+  debugf("api_spiffs_erase");
+  u32_t sect_first = flashmem_get_sector_of_address(addr);
+  u32_t sect_last = sect_first;
+  while( sect_first <= sect_last )
+    if( !flashmem_erase_sector( sect_first ++ ) )
+      return SPIFFS_ERR_INTERNAL;
+  return SPIFFS_OK;
+} 
+
+/*******************
+The W25Q32BV array is organized into 16,384 programmable pages of 256-bytes each. Up to 256 bytes can be programmed at a time.
+Pages can be erased in groups of 16 (4KB sector erase), groups of 128 (32KB block erase), groups of 256 (64KB block erase) or
+the entire chip (chip erase). The W25Q32BV has 1,024 erasable sectors and 64 erasable blocks respectively.
+The small 4KB sectors allow for greater flexibility in applications that require data and parameter storage.
+********************/
+
+extern uint32_t _SPIFFS_start;
+extern uint32_t _SPIFFS_end;
+
+spiffs_config spiffs_get_storage_config()
+{
+	spiffs_config cfg = {0};
+	cfg.phys_addr = (u32_t)&_SPIFFS_start;
+	if (cfg.phys_addr == 0)
+		return cfg;
+	cfg.phys_addr += 0x3000;
+	cfg.phys_addr &= 0xFFFFC000;  // align to 4 sector.
+	cfg.phys_size = (u32_t)((u32_t)&_SPIFFS_end - (u32_t)&_SPIFFS_start);
+	/*cfg.phys_addr = INTERNAL_FLASH_SIZE - SPIFFS_WORK_SIZE + INTERNAL_FLASH_START_ADDRESS;
+	cfg.phys_addr += 0x3000;
+	cfg.phys_addr &= 0xFFFFC000;  // align to 4 sector.
+	cfg.phys_size = SPIFFS_WORK_SIZE;*/
+	cfg.phys_erase_block = INTERNAL_FLASH_SECTOR_SIZE; // according to datasheet
+	cfg.log_block_size = INTERNAL_FLASH_SECTOR_SIZE * 2; // Important to make large
+	cfg.log_page_size = LOG_PAGE_SIZE; // as we said
+	return cfg;
+}
+
+bool spiffs_format_internal()
+{
+  spiffs_config cfg = spiffs_get_storage_config();
+  if (cfg.phys_addr == 0)
+  {
+	SYSTEM_ERROR("Can't format file system, wrong address");
+	return false;
+  }
+
+  u32_t sect_first, sect_last;
+  sect_first = cfg.phys_addr;
+  sect_first = flashmem_get_sector_of_address((u32_t)&_SPIFFS_start);
+  sect_last = flashmem_get_sector_of_address((u32_t)&_SPIFFS_end);
+  debugf("sect_first: %x, sect_last: %x\n", sect_first, sect_last);
+  while( sect_first <= sect_last )
+	if(!flashmem_erase_sector( sect_first ++ ))
+	  return false;
+}
+
+void spiffs_mount()
+{
+  spiffs_config cfg = spiffs_get_storage_config();
+  if (cfg.phys_addr == 0)
+  {
+	  SYSTEM_ERROR("Can't start file system, wrong address");
+	  return;
+  }
+
+  debugf("fs.start:%x, size:%d Kb\n", cfg.phys_addr, cfg.phys_size / 1024);
+
+  cfg.hal_read_f = api_spiffs_read;
+  cfg.hal_write_f = api_spiffs_write;
+  cfg.hal_erase_f = api_spiffs_erase;
+  
+  uint32_t dat;
+  bool writeFirst = false;
+  flashmem_read(&dat, cfg.phys_addr, 4);
+  //debugf("%X", dat);
+
+  if (dat == UINT32_MAX)
+  {
+	  debugf("First init file system");
+	  spiffs_format_internal();
+	  writeFirst = true;
+  }
+
+  int res = SPIFFS_mount(&_filesystemStorageHandle,
+    &cfg,
+    spiffs_work_buf,
+    spiffs_fds,
+    sizeof(spiffs_fds),
+    spiffs_cache,
+    sizeof(spiffs_cache),
+    NULL);
+  debugf("mount res: %d\n", res);
+
+  if (writeFirst)
+  {
+	  file_t fd = SPIFFS_open(&_filesystemStorageHandle, "initialize_fs_header.dat", SPIFFS_CREAT | SPIFFS_TRUNC | SPIFFS_RDWR, 0);
+	  SPIFFS_write(&_filesystemStorageHandle, fd, (u8_t *)"1", 1);
+	  SPIFFS_fremove(&_filesystemStorageHandle, fd);
+	  SPIFFS_close(&_filesystemStorageHandle, fd);
+  }
+
+  //dat=0;
+  //flashmem_read(&dat, cfg.phys_addr, 4);
+  //debugf("%X", dat);
+}
+
+void spiffs_unmount()
+{
+	SPIFFS_unmount(&_filesystemStorageHandle);
+}
+
+// FS formatting function
+bool spiffs_format()
+{
+  spiffs_unmount();
+  spiffs_format_internal();
+  spiffs_mount();
+  return true;
+}
+
+//int spiffs_check( void )
+//{
+  // ets_wdt_disable();
+  // int res = (int)SPIFFS_check(&_filesystemStorageHandle);
+  // ets_wdt_enable();
+  // return res;
+//}
+
+void test_spiffs()
+{
+  char buf[12] = {0};
+
+  // Surely, I've mounted spiffs before entering here
+  
+  spiffs_file fd;
+  spiffs_stat st = {0};
+  SPIFFS_stat(&_filesystemStorageHandle, "my_file.txt", &st);
+  if (st.size <= 0)
+  {
+	  fd = SPIFFS_open(&_filesystemStorageHandle, "my_file.txt", SPIFFS_CREAT | SPIFFS_TRUNC | SPIFFS_RDWR, 0);
+	  if (SPIFFS_write(&_filesystemStorageHandle, fd, (u8_t *)"Hello world", 11) < 0)
+		  debugf("errno %d\n", SPIFFS_errno(&_filesystemStorageHandle));
+	  SPIFFS_close(&_filesystemStorageHandle, fd);
+	  debugf("file created");
+  }
+  else
+	  debugf("file %s exist :)", st.name);
+
+
+  fd = SPIFFS_open(&_filesystemStorageHandle, "my_file.txt", SPIFFS_RDWR, 0);
+  if (SPIFFS_read(&_filesystemStorageHandle, fd, (u8_t *)buf, 11) < 0) debugf("errno %d\n", SPIFFS_errno(&_filesystemStorageHandle));
+  SPIFFS_close(&_filesystemStorageHandle, fd);
+
+  debugf("--> %s <--\n", buf);
+}

cores/esp8266/spiffs/spiffs.h

@@ -0,0 +1,470 @@
+/*
+ * spiffs.h
+ *
+ *  Created on: May 26, 2013
+ *      Author: petera
+ */
+
+#ifndef SPIFFS_H_
+#define SPIFFS_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+//#include "c_stdio.h"
+#include <user_config.h>
+#include "spiffs_config.h"
+#include "flashmem.h"
+
+#define SPIFFS_OK                       0
+#define SPIFFS_ERR_NOT_MOUNTED          -10000
+#define SPIFFS_ERR_FULL                 -10001
+#define SPIFFS_ERR_NOT_FOUND            -10002
+#define SPIFFS_ERR_END_OF_OBJECT        -10003
+#define SPIFFS_ERR_DELETED              -10004
+#define SPIFFS_ERR_NOT_FINALIZED        -10005
+#define SPIFFS_ERR_NOT_INDEX            -10006
+#define SPIFFS_ERR_OUT_OF_FILE_DESCS    -10007
+#define SPIFFS_ERR_FILE_CLOSED          -10008
+#define SPIFFS_ERR_FILE_DELETED         -10009
+#define SPIFFS_ERR_BAD_DESCRIPTOR       -10010
+#define SPIFFS_ERR_IS_INDEX             -10011
+#define SPIFFS_ERR_IS_FREE              -10012
+#define SPIFFS_ERR_INDEX_SPAN_MISMATCH  -10013
+#define SPIFFS_ERR_DATA_SPAN_MISMATCH   -10014
+#define SPIFFS_ERR_INDEX_REF_FREE       -10015
+#define SPIFFS_ERR_INDEX_REF_LU         -10016
+#define SPIFFS_ERR_INDEX_REF_INVALID    -10017
+#define SPIFFS_ERR_INDEX_FREE           -10018
+#define SPIFFS_ERR_INDEX_LU             -10019
+#define SPIFFS_ERR_INDEX_INVALID        -10020
+#define SPIFFS_ERR_NOT_WRITABLE         -10021
+#define SPIFFS_ERR_NOT_READABLE         -10022
+#define SPIFFS_ERR_CONFLICTING_NAME     -10023
+
+#define SPIFFS_ERR_INTERNAL             -10050
+
+#define SPIFFS_ERR_TEST                 -10100
+
+
+// spiffs file descriptor index type. must be signed
+typedef s16_t spiffs_file;
+// spiffs file descriptor flags
+typedef u16_t spiffs_flags;
+// spiffs file mode
+typedef u16_t spiffs_mode;
+// object type
+typedef u8_t spiffs_obj_type;
+
+/* spi read call function type */
+typedef s32_t (*spiffs_read)(u32_t addr, u32_t size, u8_t *dst);
+/* spi write call function type */
+typedef s32_t (*spiffs_write)(u32_t addr, u32_t size, u8_t *src);
+/* spi erase call function type */
+typedef s32_t (*spiffs_erase)(u32_t addr, u32_t size);
+
+/* file system check callback report operation */
+typedef enum {
+  SPIFFS_CHECK_LOOKUP = 0,
+  SPIFFS_CHECK_INDEX,
+  SPIFFS_CHECK_PAGE
+} spiffs_check_type;
+
+/* file system check callback report type */
+typedef enum {
+  SPIFFS_CHECK_PROGRESS = 0,
+  SPIFFS_CHECK_ERROR,
+  SPIFFS_CHECK_FIX_INDEX,
+  SPIFFS_CHECK_FIX_LOOKUP,
+  SPIFFS_CHECK_DELETE_ORPHANED_INDEX,
+  SPIFFS_CHECK_DELETE_PAGE,
+  SPIFFS_CHECK_DELETE_BAD_FILE,
+} spiffs_check_report;
+
+/* file system check callback function */
+typedef void (*spiffs_check_callback)(spiffs_check_type type, spiffs_check_report report,
+    u32_t arg1, u32_t arg2);
+
+#ifndef SPIFFS_DBG
+#define SPIFFS_DBG(...) \
+    print(__VA_ARGS__)
+#endif
+#ifndef SPIFFS_GC_DBG
+#define SPIFFS_GC_DBG(...) printf(__VA_ARGS__)
+#endif
+#ifndef SPIFFS_CACHE_DBG
+#define SPIFFS_CACHE_DBG(...) printf(__VA_ARGS__)
+#endif
+#ifndef SPIFFS_CHECK_DBG
+#define SPIFFS_CHECK_DBG(...) printf(__VA_ARGS__)
+#endif
+
+/* Any write to the filehandle is appended to end of the file */
+#define SPIFFS_APPEND                   (1<<0)
+/* If the opened file exists, it will be truncated to zero length before opened */
+#define SPIFFS_TRUNC                    (1<<1)
+/* If the opened file does not exist, it will be created before opened */
+#define SPIFFS_CREAT                    (1<<2)
+/* The opened file may only be read */
+#define SPIFFS_RDONLY                   (1<<3)
+/* The opened file may only be writted */
+#define SPIFFS_WRONLY                   (1<<4)
+/* The opened file may be both read and writted */
+#define SPIFFS_RDWR                     (SPIFFS_RDONLY | SPIFFS_WRONLY)
+/* Any writes to the filehandle will never be cached */
+#define SPIFFS_DIRECT                   (1<<5)
+
+#define SPIFFS_SEEK_SET                 (0)
+#define SPIFFS_SEEK_CUR                 (1)
+#define SPIFFS_SEEK_END                 (2)
+
+#define SPIFFS_TYPE_FILE                (1)
+#define SPIFFS_TYPE_DIR                 (2)
+#define SPIFFS_TYPE_HARD_LINK           (3)
+#define SPIFFS_TYPE_SOFT_LINK           (4)
+
+#ifndef SPIFFS_LOCK
+#define SPIFFS_LOCK(fs)
+#endif
+
+#ifndef SPIFFS_UNLOCK
+#define SPIFFS_UNLOCK(fs)
+#endif
+
+// phys structs
+
+// spiffs spi configuration struct
+typedef struct {
+  // physical read function
+  spiffs_read hal_read_f;
+  // physical write function
+  spiffs_write hal_write_f;
+  // physical erase function
+  spiffs_erase hal_erase_f;
+#if SPIFFS_SINGLETON == 0
+  // physical size of the spi flash
+  u32_t phys_size;
+  // physical offset in spi flash used for spiffs,
+  // must be on block boundary
+  u32_t phys_addr;
+  // physical size when erasing a block
+  u32_t phys_erase_block;
+
+  // logical size of a block, must be on physical
+  // block size boundary and must never be less than
+  // a physical block
+  u32_t log_block_size;
+  // logical size of a page, must be at least
+  // log_block_size / 8
+  u32_t log_page_size;
+#endif
+} spiffs_config;
+
+typedef struct {
+  // file system configuration
+  spiffs_config cfg;
+  // number of logical blocks
+  u32_t block_count;
+
+  // cursor for free blocks, block index
+  spiffs_block_ix free_cursor_block_ix;
+  // cursor for free blocks, entry index
+  int free_cursor_obj_lu_entry;
+  // cursor when searching, block index
+  spiffs_block_ix cursor_block_ix;
+  // cursor when searching, entry index
+  int cursor_obj_lu_entry;
+
+  // primary work buffer, size of a logical page
+  u8_t *lu_work;
+  // secondary work buffer, size of a logical page
+  u8_t *work;
+  // file descriptor memory area
+  u8_t *fd_space;
+  // available file descriptors
+  u32_t fd_count;
+
+  // last error
+  s32_t errno;
+
+  // current number of free blocks
+  u32_t free_blocks;
+  // current number of busy pages
+  u32_t stats_p_allocated;
+  // current number of deleted pages
+  u32_t stats_p_deleted;
+  // flag indicating that garbage collector is cleaning
+  u8_t cleaning;
+  // max erase count amongst all blocks
+  spiffs_obj_id max_erase_count;
+
+#if SPIFFS_GC_STATS
+  u32_t stats_gc_runs;
+#endif
+
+#if SPIFFS_CACHE
+  // cache memory
+  void *cache;
+  // cache size
+  u32_t cache_size;
+#if SPIFFS_CACHE_STATS
+  u32_t cache_hits;
+  u32_t cache_misses;
+#endif
+#endif
+
+  // check callback function
+  spiffs_check_callback check_cb_f;
+} spiffs;
+
+/* spiffs file status struct */
+typedef struct {
+  spiffs_obj_id obj_id;
+  u32_t size;
+  spiffs_obj_type type;
+  u8_t name[SPIFFS_OBJ_NAME_LEN];
+} spiffs_stat;
+
+struct spiffs_dirent {
+  spiffs_obj_id obj_id;
+  u8_t name[SPIFFS_OBJ_NAME_LEN];
+  spiffs_obj_type type;
+  u32_t size;
+  spiffs_page_ix pix;
+};
+
+typedef struct {
+  spiffs *fs;
+  spiffs_block_ix block;
+  int entry;
+} spiffs_DIR;
+
+// functions
+
+/**
+ * Initializes the file system dynamic parameters and mounts the filesystem
+ * @param fs            the file system struct
+ * @param config        the physical and logical configuration of the file system
+ * @param work          a memory work buffer comprising 2*config->log_page_size
+ *                      bytes used throughout all file system operations
+ * @param fd_space      memory for file descriptors
+ * @param fd_space_size memory size of file descriptors
+ * @param cache         memory for cache, may be null
+ * @param cache_size    memory size of cache
+ * @param check_cb_f    callback function for reporting during consistency checks
+ */
+s32_t SPIFFS_mount(spiffs *fs, spiffs_config *config, u8_t *work,
+    u8_t *fd_space, u32_t fd_space_size,
+    void *cache, u32_t cache_size,
+    spiffs_check_callback check_cb_f);
+
+/**
+ * Unmounts the file system. All file handles will be flushed of any
+ * cached writes and closed.
+ * @param fs            the file system struct
+ */
+void SPIFFS_unmount(spiffs *fs);
+
+/**
+ * Creates a new file.
+ * @param fs            the file system struct
+ * @param path          the path of the new file
+ * @param mode          ignored, for posix compliance
+ */
+s32_t SPIFFS_creat(spiffs *fs, const char *path, spiffs_mode mode);
+
+/**
+ * Opens/creates a file.
+ * @param fs            the file system struct
+ * @param path          the path of the new file
+ * @param flags         the flags for the open command, can be combinations of
+ *                      SPIFFS_APPEND, SPIFFS_TRUNC, SPIFFS_CREAT, SPIFFS_RD_ONLY,
+ *                      SPIFFS_WR_ONLY, SPIFFS_RDWR, SPIFFS_DIRECT
+ * @param mode          ignored, for posix compliance
+ */
+spiffs_file SPIFFS_open(spiffs *fs, const char *path, spiffs_flags flags, spiffs_mode mode);
+
+/**
+ * Opens a file by given dir entry.
+ * Optimization purposes, when traversing a file system with SPIFFS_readdir
+ * a normal SPIFFS_open would need to traverse the filesystem again to find
+ * the file, whilst SPIFFS_open_by_dirent already knows where the file resides.
+ * @param fs            the file system struct
+ * @param path          the dir entry to the file
+ * @param flags         the flags for the open command, can be combinations of
+ *                      SPIFFS_APPEND, SPIFFS_TRUNC, SPIFFS_CREAT, SPIFFS_RD_ONLY,
+ *                      SPIFFS_WR_ONLY, SPIFFS_RDWR, SPIFFS_DIRECT.
+ *                      SPIFFS_CREAT will have no effect in this case.
+ * @param mode          ignored, for posix compliance
+ */
+spiffs_file SPIFFS_open_by_dirent(spiffs *fs, struct spiffs_dirent *e, spiffs_flags flags, spiffs_mode mode);
+
+/**
+ * Reads from given filehandle.
+ * @param fs            the file system struct
+ * @param fh            the filehandle
+ * @param buf           where to put read data
+ * @param len           how much to read
+ * @returns number of bytes read, or -1 if error
+ */
+s32_t SPIFFS_read(spiffs *fs, spiffs_file fh, void *buf, u32_t len);
+
+/**
+ * Writes to given filehandle.
+ * @param fs            the file system struct
+ * @param fh            the filehandle
+ * @param buf           the data to write
+ * @param len           how much to write
+ * @returns number of bytes written, or -1 if error
+ */
+s32_t SPIFFS_write(spiffs *fs, spiffs_file fh, void *buf, u32_t len);
+
+/**
+ * Moves the read/write file offset
+ * @param fs            the file system struct
+ * @param fh            the filehandle
+ * @param offs          how much/where to move the offset
+ * @param whence        if SPIFFS_SEEK_SET, the file offset shall be set to offset bytes
+ *                      if SPIFFS_SEEK_CUR, the file offset shall be set to its current location plus offset
+ *                      if SPIFFS_SEEK_END, the file offset shall be set to the size of the file plus offset
+ */
+s32_t SPIFFS_lseek(spiffs *fs, spiffs_file fh, s32_t offs, int whence);
+
+/**
+ * Removes a file by path
+ * @param fs            the file system struct
+ * @param path          the path of the file to remove
+ */
+s32_t SPIFFS_remove(spiffs *fs, const char *path);
+
+/**
+ * Removes a file by filehandle
+ * @param fs            the file system struct
+ * @param fh            the filehandle of the file to remove
+ */
+s32_t SPIFFS_fremove(spiffs *fs, spiffs_file fh);
+
+/**
+ * Gets file status by path
+ * @param fs            the file system struct
+ * @param path          the path of the file to stat
+ * @param s             the stat struct to populate
+ */
+s32_t SPIFFS_stat(spiffs *fs, const char *path, spiffs_stat *s);
+
+/**
+ * Gets file status by filehandle
+ * @param fs            the file system struct
+ * @param fh            the filehandle of the file to stat
+ * @param s             the stat struct to populate
+ */
+s32_t SPIFFS_fstat(spiffs *fs, spiffs_file fh, spiffs_stat *s);
+
+/**
+ * Flushes all pending write operations from cache for given file
+ * @param fs            the file system struct
+ * @param fh            the filehandle of the file to flush
+ */
+s32_t SPIFFS_fflush(spiffs *fs, spiffs_file fh);
+
+/**
+ * Closes a filehandle. If there are pending write operations, these are finalized before closing.
+ * @param fs            the file system struct
+ * @param fh            the filehandle of the file to close
+ */
+void SPIFFS_close(spiffs *fs, spiffs_file fh);
+
+/**
+ * Renames a file
+ * @param fs            the file system struct
+ * @param old           path of file to rename
+ * @param new           new path of file
+ */
+s32_t SPIFFS_rename(spiffs *fs, const char *old, const char *newname);
+
+/**
+ * Returns last error of last file operation.
+ * @param fs            the file system struct
+ */
+s32_t SPIFFS_errno(spiffs *fs);
+
+/**
+ * Opens a directory stream corresponding to the given name.
+ * The stream is positioned at the first entry in the directory.
+ * On hydrogen builds the name argument is ignored as hydrogen builds always correspond
+ * to a flat file structure - no directories.
+ * @param fs            the file system struct
+ * @param name          the name of the directory
+ * @param d             pointer the directory stream to be populated
+ */
+spiffs_DIR *SPIFFS_opendir(spiffs *fs, const char *name, spiffs_DIR *d);
+
+/**
+ * Closes a directory stream
+ * @param d             the directory stream to close
+ */
+s32_t SPIFFS_closedir(spiffs_DIR *d);
+
+/**
+ * Reads a directory into given spifs_dirent struct.
+ * @param d             pointer to the directory stream
+ * @param e             the dirent struct to be populated
+ * @returns null if error or end of stream, else given dirent is returned
+ */
+struct spiffs_dirent *SPIFFS_readdir(spiffs_DIR *d, struct spiffs_dirent *e);
+
+/**
+ * Runs a consistency check on given filesystem.
+ * @param fs            the file system struct
+ */
+s32_t SPIFFS_check(spiffs *fs);
+
+/**
+ * Check if EOF reached.
+ * @param fs            the file system struct
+ * @param fh            the filehandle of the file to check
+ */
+s32_t SPIFFS_eof(spiffs *fs, spiffs_file fh);
+s32_t SPIFFS_tell(spiffs *fs, spiffs_file fh);
+
+#if SPIFFS_TEST_VISUALISATION
+/**
+ * Prints out a visualization of the filesystem.
+ * @param fs            the file system struct
+ */
+s32_t SPIFFS_vis(spiffs *fs);
+#endif
+
+#if SPIFFS_BUFFER_HELP
+/**
+ * Returns number of bytes needed for the filedescriptor buffer given
+ * amount of file descriptors.
+ */
+u32_t SPIFFS_buffer_bytes_for_filedescs(spiffs *fs, u32_t num_descs);
+
+#if SPIFFS_CACHE
+/**
+ * Returns number of bytes needed for the cache buffer given
+ * amount of cache pages.
+ */
+u32_t SPIFFS_buffer_bytes_for_cache(spiffs *fs, u32_t num_pages);
+#endif
+#endif
+
+#if SPIFFS_CACHE
+#endif
+
+
+void spiffs_mount();
+void spiffs_unmount();
+bool spiffs_format();
+spiffs_config spiffs_get_storage_config();
+extern void test_spiffs();
+
+extern spiffs _filesystemStorageHandle;
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* SPIFFS_H_ */

cores/esp8266/spiffs/spiffs_cache.c

@@ -0,0 +1,303 @@
+/*
+ * spiffs_cache.c
+ *
+ *  Created on: Jun 23, 2013
+ *      Author: petera
+ */
+
+#include "spiffs.h"
+#include "spiffs_nucleus.h"
+
+#if SPIFFS_CACHE
+
+// returns cached page for give page index, or null if no such cached page
+static spiffs_cache_page *spiffs_cache_page_get(spiffs *fs, spiffs_page_ix pix) {
+  spiffs_cache *cache = spiffs_get_cache(fs);
+  if ((cache->cpage_use_map & cache->cpage_use_mask) == 0) return 0;
+  int i;
+  for (i = 0; i < cache->cpage_count; i++) {
+    spiffs_cache_page *cp = spiffs_get_cache_page_hdr(fs, cache, i);
+    if ((cache->cpage_use_map & (1<<i)) &&
+        (cp->flags & SPIFFS_CACHE_FLAG_TYPE_WR) == 0 &&
+        cp->pix == pix ) {
+      SPIFFS_CACHE_DBG("CACHE_GET: have cache page %u for %04x\n", i, pix);
+      cp->last_access = cache->last_access;
+      return cp;
+    }
+  }
+  //SPIFFS_CACHE_DBG("CACHE_GET: no cache for %04x\n", pix);
+  return 0;
+}
+
+// frees cached page
+static s32_t spiffs_cache_page_free(spiffs *fs, int ix, u8_t write_back) {
+  s32_t res = SPIFFS_OK;
+  spiffs_cache *cache = spiffs_get_cache(fs);
+  spiffs_cache_page *cp = spiffs_get_cache_page_hdr(fs, cache, ix);
+  if (cache->cpage_use_map & (1<<ix)) {
+    if (write_back &&
+        (cp->flags & SPIFFS_CACHE_FLAG_TYPE_WR) == 0 &&
+        (cp->flags & SPIFFS_CACHE_FLAG_DIRTY)) {
+      u8_t *mem =  spiffs_get_cache_page(fs, cache, ix);
+      res = fs->cfg.hal_write_f(SPIFFS_PAGE_TO_PADDR(fs, cp->pix), SPIFFS_CFG_LOG_PAGE_SZ(fs), mem);
+    }
+
+    cp->flags = 0;
+    cache->cpage_use_map &= ~(1 << ix);
+
+    if (cp->flags & SPIFFS_CACHE_FLAG_TYPE_WR) {
+      SPIFFS_CACHE_DBG("CACHE_FREE: free cache page %u objid %04x\n", ix, cp->obj_id);
+    } else {
+      SPIFFS_CACHE_DBG("CACHE_FREE: free cache page %u pix %04x\n", ix, cp->pix);
+    }
+  }
+
+  return res;
+}
+
+// removes the oldest accessed cached page
+static s32_t spiffs_cache_page_remove_oldest(spiffs *fs, u8_t flag_mask, u8_t flags) {
+  s32_t res = SPIFFS_OK;
+  spiffs_cache *cache = spiffs_get_cache(fs);
+
+  if ((cache->cpage_use_map & cache->cpage_use_mask) != cache->cpage_use_mask) {
+    // at least one free cpage
+    return SPIFFS_OK;
+  }
+
+  // all busy, scan thru all to find the cpage which has oldest access
+  int i;
+  int cand_ix = -1;
+  u32_t oldest_val = 0;
+  for (i = 0; i < cache->cpage_count; i++) {
+    spiffs_cache_page *cp = spiffs_get_cache_page_hdr(fs, cache, i);
+    if ((cache->last_access - cp->last_access) > oldest_val &&
+        (cp->flags & flag_mask) == flags) {
+      oldest_val = cache->last_access - cp->last_access;
+      cand_ix = i;
+    }
+  }
+
+  if (cand_ix >= 0) {
+    res = spiffs_cache_page_free(fs, cand_ix, 1);
+  }
+
+  return res;
+}
+
+// allocates a new cached page and returns it, or null if all cache pages are busy
+static spiffs_cache_page *spiffs_cache_page_allocate(spiffs *fs) {
+  spiffs_cache *cache = spiffs_get_cache(fs);
+  if (cache->cpage_use_map == 0xffffffff) {
+    // out of cache memory
+    return 0;
+  }
+  int i;
+  for (i = 0; i < cache->cpage_count; i++) {
+    if ((cache->cpage_use_map & (1<<i)) == 0) {
+      spiffs_cache_page *cp = spiffs_get_cache_page_hdr(fs, cache, i);
+      cache->cpage_use_map |= (1<<i);
+      cp->last_access = cache->last_access;
+      SPIFFS_CACHE_DBG("CACHE_ALLO: allocated cache page %u\n", i);
+      return cp;
+    }
+  }
+  // out of cache entries
+  return 0;
+}
+
+// drops the cache page for give page index
+void spiffs_cache_drop_page(spiffs *fs, spiffs_page_ix pix) {
+  spiffs_cache_page *cp =  spiffs_cache_page_get(fs, pix);
+  if (cp) {
+    spiffs_cache_page_free(fs, cp->ix, 0);
+  }
+}
+
+// ------------------------------
+
+// reads from spi flash or the cache
+s32_t spiffs_phys_rd(
+    spiffs *fs,
+    u8_t op,
+    spiffs_file fh,
+    u32_t addr,
+    u32_t len,
+    u8_t *dst) {
+  (void)fh;
+  s32_t res = SPIFFS_OK;
+  spiffs_cache *cache = spiffs_get_cache(fs);
+  spiffs_cache_page *cp =  spiffs_cache_page_get(fs, SPIFFS_PADDR_TO_PAGE(fs, addr));
+  cache->last_access++;
+  if (cp) {
+#if SPIFFS_CACHE_STATS
+    fs->cache_hits++;
+#endif
+    cp->last_access = cache->last_access;
+  } else {
+    if ((op & SPIFFS_OP_TYPE_MASK) == SPIFFS_OP_T_OBJ_LU2) {
+      // for second layer lookup functions, we do not cache in order to prevent shredding
+      return fs->cfg.hal_read_f(
+          addr ,
+          len,
+          dst);
+    }
+#if SPIFFS_CACHE_STATS
+    fs->cache_misses++;
+#endif
+    res = spiffs_cache_page_remove_oldest(fs, SPIFFS_CACHE_FLAG_TYPE_WR, 0);
+    cp = spiffs_cache_page_allocate(fs);
+    if (cp) {
+      cp->flags = SPIFFS_CACHE_FLAG_WRTHRU;
+      cp->pix = SPIFFS_PADDR_TO_PAGE(fs, addr);
+    }
+
+    s32_t res2 = fs->cfg.hal_read_f(
+        addr - SPIFFS_PADDR_TO_PAGE_OFFSET(fs, addr),
+        SPIFFS_CFG_LOG_PAGE_SZ(fs),
+        spiffs_get_cache_page(fs, cache, cp->ix));
+    if (res2 != SPIFFS_OK) {
+      res = res2;
+    }
+  }
+  u8_t *mem =  spiffs_get_cache_page(fs, cache, cp->ix);
+  c_memcpy(dst, &mem[SPIFFS_PADDR_TO_PAGE_OFFSET(fs, addr)], len);
+  return res;
+}
+
+// writes to spi flash and/or the cache
+s32_t spiffs_phys_wr(
+    spiffs *fs,
+    u8_t op,
+    spiffs_file fh,
+    u32_t addr,
+    u32_t len,
+    u8_t *src) {
+  (void)fh;
+  spiffs_page_ix pix = SPIFFS_PADDR_TO_PAGE(fs, addr);
+  spiffs_cache *cache = spiffs_get_cache(fs);
+  spiffs_cache_page *cp =  spiffs_cache_page_get(fs, pix);
+
+  if (cp && (op & SPIFFS_OP_COM_MASK) != SPIFFS_OP_C_WRTHRU) {
+    // have a cache page
+    // copy in data to cache page
+
+    if ((op & SPIFFS_OP_COM_MASK) == SPIFFS_OP_C_DELE &&
+        (op & SPIFFS_OP_TYPE_MASK) != SPIFFS_OP_T_OBJ_LU) {
+      // page is being deleted, wipe from cache - unless it is a lookup page
+      spiffs_cache_page_free(fs, cp->ix, 0);
+      return fs->cfg.hal_write_f(addr, len, src);
+    }
+
+    u8_t *mem =  spiffs_get_cache_page(fs, cache, cp->ix);
+    c_memcpy(&mem[SPIFFS_PADDR_TO_PAGE_OFFSET(fs, addr)], src, len);
+
+    cache->last_access++;
+    cp->last_access = cache->last_access;
+
+    if (cp->flags && SPIFFS_CACHE_FLAG_WRTHRU) {
+      // page is being updated, no write-cache, just pass thru
+      return fs->cfg.hal_write_f(addr, len, src);
+    } else {
+      return SPIFFS_OK;
+    }
+  } else {
+    // no cache page, no write cache - just write thru
+    return fs->cfg.hal_write_f(addr, len, src);
+  }
+}
+
+#if SPIFFS_CACHE_WR
+// returns the cache page that this fd refers, or null if no cache page
+spiffs_cache_page *spiffs_cache_page_get_by_fd(spiffs *fs, spiffs_fd *fd) {
+  spiffs_cache *cache = spiffs_get_cache(fs);
+
+  if ((cache->cpage_use_map & cache->cpage_use_mask) == 0) {
+    // all cpages free, no cpage cannot be assigned to obj_id
+    return 0;
+  }
+
+  int i;
+  for (i = 0; i < cache->cpage_count; i++) {
+    spiffs_cache_page *cp = spiffs_get_cache_page_hdr(fs, cache, i);
+    if ((cache->cpage_use_map & (1<<i)) &&
+        (cp->flags & SPIFFS_CACHE_FLAG_TYPE_WR) &&
+        cp->obj_id == fd->obj_id) {
+      return cp;
+    }
+  }
+
+  return 0;
+}
+
+// allocates a new cache page and refers this to given fd - flushes an old cache
+// page if all cache is busy
+spiffs_cache_page *spiffs_cache_page_allocate_by_fd(spiffs *fs, spiffs_fd *fd) {
+  // before this function is called, it is ensured that there is no already existing
+  // cache page with same object id
+  spiffs_cache_page_remove_oldest(fs, SPIFFS_CACHE_FLAG_TYPE_WR, 0);
+  spiffs_cache_page *cp = spiffs_cache_page_allocate(fs);
+  if (cp == 0) {
+    // could not get cache page
+    return 0;
+  }
+
+  cp->flags = SPIFFS_CACHE_FLAG_TYPE_WR;
+  cp->obj_id = fd->obj_id;
+  fd->cache_page = cp;
+  return cp;
+}
+
+// unrefers all fds that this cache page refers to and releases the cache page
+void spiffs_cache_fd_release(spiffs *fs, spiffs_cache_page *cp) {
+  if (cp == 0) return;
+  u32_t i;
+  spiffs_fd *fds = (spiffs_fd *)fs->fd_space;
+  for (i = 0; i < fs->fd_count; i++) {
+    spiffs_fd *cur_fd = &fds[i];
+    if (cur_fd->file_nbr != 0 && cur_fd->cache_page == cp) {
+      cur_fd->cache_page = 0;
+    }
+  }
+  spiffs_cache_page_free(fs, cp->ix, 0);
+
+  cp->obj_id = 0;
+}
+
+#endif
+
+// initializes the cache
+void spiffs_cache_init(spiffs *fs) {
+  if (fs->cache == 0) return;
+  u32_t sz = fs->cache_size;
+  u32_t cache_mask = 0;
+  int i;
+  int cache_entries =
+      (sz - sizeof(spiffs_cache)) / (SPIFFS_CACHE_PAGE_SIZE(fs));
+  if (cache_entries <= 0) return;
+
+  for (i = 0; i < cache_entries; i++) {
+    cache_mask <<= 1;
+    cache_mask |= 1;
+  }
+
+  spiffs_cache cache;
+  c_memset(&cache, 0, sizeof(spiffs_cache));
+  cache.cpage_count = cache_entries;
+  cache.cpages = (u8_t *)((u8_t *)fs->cache + sizeof(spiffs_cache));
+
+  cache.cpage_use_map = 0xffffffff;
+  cache.cpage_use_mask = cache_mask;
+  c_memcpy(fs->cache, &cache, sizeof(spiffs_cache));
+
+  spiffs_cache *c = spiffs_get_cache(fs);
+
+  c_memset(c->cpages, 0, c->cpage_count * SPIFFS_CACHE_PAGE_SIZE(fs));
+
+  c->cpage_use_map &= ~(c->cpage_use_mask);
+  for (i = 0; i < cache.cpage_count; i++) {
+    spiffs_get_cache_page_hdr(fs, c, i)->ix = i;
+  }
+}
+
+#endif // SPIFFS_CACHE

cores/esp8266/spiffs/spiffs_check.c

@@ -0,0 +1,973 @@
+/*
+ * spiffs_check.c
+ *
+ * Contains functionality for checking file system consistency
+ * and mending problems.
+ * Three levels of consistency checks are implemented:
+ *
+ * Look up consistency
+ *   Checks if indices in lookup pages are coherent with page headers
+ * Object index consistency
+ *   Checks if there are any orphaned object indices (missing object index headers).
+ *   If an object index is found but not its header, the object index is deleted.
+ *   This is critical for the following page consistency check.
+ * Page consistency
+ *   Checks for pages that ought to be indexed, ought not to be indexed, are multiple indexed
+ *
+ *
+ *  Created on: Jul 7, 2013
+ *      Author: petera
+ */
+
+#include "spiffs.h"
+#include "spiffs_nucleus.h"
+
+//---------------------------------------
+// Look up consistency
+
+// searches in the object indices and returns the referenced page index given
+// the object id and the data span index
+// destroys fs->lu_work
+static s32_t spiffs_object_get_data_page_index_reference(
+  spiffs *fs,
+  spiffs_obj_id obj_id,
+  spiffs_span_ix data_spix,
+  spiffs_page_ix *pix,
+  spiffs_page_ix *objix_pix) {
+  s32_t res;
+
+  // calculate object index span index for given data page span index
+  spiffs_span_ix objix_spix = SPIFFS_OBJ_IX_ENTRY_SPAN_IX(fs, data_spix);
+
+  // find obj index for obj id and span index
+  res = spiffs_obj_lu_find_id_and_span(fs, obj_id | SPIFFS_OBJ_ID_IX_FLAG, objix_spix, 0, objix_pix);
+  SPIFFS_CHECK_RES(res);
+
+  // load obj index entry
+  u32_t addr = SPIFFS_PAGE_TO_PADDR(fs, *objix_pix);
+  if (objix_spix == 0) {
+    // get referenced page from object index header
+    addr += sizeof(spiffs_page_object_ix_header) + data_spix * sizeof(spiffs_page_ix);
+  } else {
+    // get referenced page from object index
+    addr += sizeof(spiffs_page_object_ix) + SPIFFS_OBJ_IX_ENTRY(fs, data_spix) * sizeof(spiffs_page_ix);
+  }
+
+  res = _spiffs_rd(fs, SPIFFS_OP_T_OBJ_LU2 | SPIFFS_OP_C_READ, 0, addr, sizeof(spiffs_page_ix), (u8_t *)pix);
+
+  return res;
+}
+
+// copies page contents to a new page
+static s32_t spiffs_rewrite_page(spiffs *fs, spiffs_page_ix cur_pix, spiffs_page_header *p_hdr, spiffs_page_ix *new_pix) {
+  s32_t res;
+  res = spiffs_page_allocate_data(fs, p_hdr->obj_id, p_hdr, 0,0,0,0, new_pix);
+  SPIFFS_CHECK_RES(res);
+  res = spiffs_phys_cpy(fs, 0,
+      SPIFFS_PAGE_TO_PADDR(fs, *new_pix) + sizeof(spiffs_page_header),
+      SPIFFS_PAGE_TO_PADDR(fs, cur_pix) + sizeof(spiffs_page_header),
+      SPIFFS_DATA_PAGE_SIZE(fs));
+  SPIFFS_CHECK_RES(res);
+  return res;
+}
+
+// rewrites the object index for given object id and replaces the
+// data page index to a new page index
+static s32_t spiffs_rewrite_index(spiffs *fs, spiffs_obj_id obj_id, spiffs_span_ix data_spix, spiffs_page_ix new_data_pix, spiffs_page_ix objix_pix) {
+  s32_t res;
+  spiffs_block_ix bix;
+  int entry;
+  spiffs_page_ix free_pix;
+  obj_id |= SPIFFS_OBJ_ID_IX_FLAG;
+
+  // find free entry
+  res = spiffs_obj_lu_find_free(fs, fs->free_cursor_block_ix, fs->free_cursor_obj_lu_entry, &bix, &entry);
+  SPIFFS_CHECK_RES(res);
+  free_pix = SPIFFS_OBJ_LOOKUP_ENTRY_TO_PIX(fs, bix, entry);
+
+  // calculate object index span index for given data page span index
+  spiffs_span_ix objix_spix = SPIFFS_OBJ_IX_ENTRY_SPAN_IX(fs, data_spix);
+  if (objix_spix == 0) {
+    // calc index in index header
+    entry = data_spix;
+  } else {
+    // calc entry in index
+    entry = SPIFFS_OBJ_IX_ENTRY(fs, data_spix);
+  }
+  // load index
+  res = _spiffs_rd(fs, SPIFFS_OP_T_OBJ_LU2 | SPIFFS_OP_C_READ,
+      0, SPIFFS_PAGE_TO_PADDR(fs, objix_pix), SPIFFS_CFG_LOG_PAGE_SZ(fs), fs->lu_work);
+  SPIFFS_CHECK_RES(res);
+  spiffs_page_header *objix_p_hdr = (spiffs_page_header *)fs->lu_work;
+
+  // be ultra safe, double check header against provided data
+  if (objix_p_hdr->obj_id != obj_id) {
+    spiffs_page_delete(fs, free_pix);
+    return SPIFFS_ERR_CHECK_OBJ_ID_MISM;
+  }
+  if (objix_p_hdr->span_ix != objix_spix) {
+    spiffs_page_delete(fs, free_pix);
+    return SPIFFS_ERR_CHECK_SPIX_MISM;
+  }
+  if ((objix_p_hdr->flags & (SPIFFS_PH_FLAG_USED | SPIFFS_PH_FLAG_IXDELE | SPIFFS_PH_FLAG_INDEX |
+                            SPIFFS_PH_FLAG_FINAL | SPIFFS_PH_FLAG_DELET)) !=
+                                (SPIFFS_PH_FLAG_IXDELE | SPIFFS_PH_FLAG_DELET)) {
+    spiffs_page_delete(fs, free_pix);
+    return SPIFFS_ERR_CHECK_FLAGS_BAD;
+  }
+
+  // rewrite in mem
+  if (objix_spix == 0) {
+    ((spiffs_page_ix*)((u8_t *)fs->lu_work + sizeof(spiffs_page_object_ix_header)))[data_spix] = new_data_pix;
+  } else {
+    ((spiffs_page_ix*)((u8_t *)fs->lu_work + sizeof(spiffs_page_object_ix)))[SPIFFS_OBJ_IX_ENTRY(fs, data_spix)] = new_data_pix;
+  }
+
+  res = _spiffs_wr(fs, SPIFFS_OP_T_OBJ_DA | SPIFFS_OP_C_UPDT,
+      0, SPIFFS_PAGE_TO_PADDR(fs, free_pix), SPIFFS_CFG_LOG_PAGE_SZ(fs), fs->lu_work);
+  SPIFFS_CHECK_RES(res);
+  res = _spiffs_wr(fs, SPIFFS_OP_T_OBJ_LU | SPIFFS_OP_C_UPDT,
+      0, SPIFFS_BLOCK_TO_PADDR(fs, SPIFFS_BLOCK_FOR_PAGE(fs, free_pix)) + SPIFFS_OBJ_LOOKUP_ENTRY_FOR_PAGE(fs, free_pix) * sizeof(spiffs_page_ix),
+      sizeof(spiffs_obj_id),
+      (u8_t *)&obj_id);
+  SPIFFS_CHECK_RES(res);
+  res = spiffs_page_delete(fs, objix_pix);
+
+  return res;
+}
+
+// deletes an object just by marking object index header as deleted
+static s32_t spiffs_delete_obj_lazy(spiffs *fs, spiffs_obj_id obj_id) {
+  spiffs_page_ix objix_hdr_pix;
+  s32_t res;
+  res = spiffs_obj_lu_find_id_and_span(fs, obj_id, 0, 0, &objix_hdr_pix);
+  if (res == SPIFFS_ERR_NOT_FOUND) {
+    return SPIFFS_OK;
+  }
+  SPIFFS_CHECK_RES(res);
+  u8_t flags = 0xff & ~SPIFFS_PH_FLAG_IXDELE;
+  res = _spiffs_wr(fs, SPIFFS_OP_T_OBJ_LU | SPIFFS_OP_C_UPDT,
+      0, SPIFFS_PAGE_TO_PADDR(fs, objix_hdr_pix) + offsetof(spiffs_page_header, flags),
+      sizeof(u8_t),
+      (u8_t *)&flags);
+  return res;
+}
+
+// validates the given look up entry
+static s32_t spiffs_lookup_check_validate(spiffs *fs, spiffs_obj_id lu_obj_id, spiffs_page_header *p_hdr,
+    spiffs_page_ix cur_pix, spiffs_block_ix cur_block, int cur_entry, int *reload_lu) {
+  (void)cur_block;
+  (void)cur_entry;
+  u8_t delete_page = 0;
+  s32_t res = SPIFFS_OK;
+  spiffs_page_ix objix_pix;
+  spiffs_page_ix ref_pix;
+  // check validity, take actions
+  if (((lu_obj_id == SPIFFS_OBJ_ID_DELETED) && (p_hdr->flags & SPIFFS_PH_FLAG_DELET)) ||
+      ((lu_obj_id == SPIFFS_OBJ_ID_FREE) && (p_hdr->flags & SPIFFS_PH_FLAG_USED) == 0)) {
+    // look up entry deleted / free but used in page header
+    SPIFFS_CHECK_DBG("LU: pix %04x deleted/free in lu but not on page\n", cur_pix);
+    *reload_lu = 1;
+    delete_page = 1;
+    if (p_hdr->flags & SPIFFS_PH_FLAG_INDEX) {
+      // header says data page
+      // data page can be removed if not referenced by some object index
+      res = spiffs_object_get_data_page_index_reference(fs, p_hdr->obj_id, p_hdr->span_ix, &ref_pix, &objix_pix);
+      if (res == SPIFFS_ERR_NOT_FOUND) {
+        // no object with this id, so remove page safely
+        res = SPIFFS_OK;
+      } else {
+        SPIFFS_CHECK_RES(res);
+        if (ref_pix == cur_pix) {
+          // data page referenced by object index but deleted in lu
+          // copy page to new place and re-write the object index to new place
+          spiffs_page_ix new_pix;
+          res = spiffs_rewrite_page(fs, cur_pix, p_hdr, &new_pix);
+          SPIFFS_CHECK_DBG("LU: FIXUP: data page not found elsewhere, rewriting %04x to new page %04x\n", cur_pix, new_pix);
+          SPIFFS_CHECK_RES(res);
+          *reload_lu = 1;
+          SPIFFS_CHECK_DBG("LU: FIXUP: %04x rewritten to %04x, affected objix_pix %04x\n", cur_pix, new_pix, objix_pix);
+          res = spiffs_rewrite_index(fs, p_hdr->obj_id, p_hdr->span_ix, new_pix, objix_pix);
+          if (res <= _SPIFFS_ERR_CHECK_FIRST && res > _SPIFFS_ERR_CHECK_LAST) {
+            // index bad also, cannot mend this file
+            SPIFFS_CHECK_DBG("LU: FIXUP: index bad %d, cannot mend!\n", res);
+            res = spiffs_page_delete(fs, new_pix);
+            SPIFFS_CHECK_RES(res);
+            res = spiffs_delete_obj_lazy(fs, p_hdr->obj_id);
+            if (fs->check_cb_f) fs->check_cb_f(SPIFFS_CHECK_LOOKUP, SPIFFS_CHECK_DELETE_BAD_FILE, p_hdr->obj_id, 0);
+          } else {
+            if (fs->check_cb_f) fs->check_cb_f(SPIFFS_CHECK_LOOKUP, SPIFFS_CHECK_FIX_INDEX, p_hdr->obj_id, p_hdr->span_ix);
+          }
+          SPIFFS_CHECK_RES(res);
+        }
+      }
+    } else {
+      // header says index page
+      // index page can be removed if other index with same obj_id and spanix is found
+      res = spiffs_obj_lu_find_id_and_span(fs, p_hdr->obj_id | SPIFFS_OBJ_ID_IX_FLAG, p_hdr->span_ix, cur_pix, 0);
+      if (res == SPIFFS_ERR_NOT_FOUND) {
+        // no such index page found, check for a data page amongst page headers
+        // lu cannot be trusted
+        res = spiffs_obj_lu_find_id_and_span_by_phdr(fs, p_hdr->obj_id | SPIFFS_OBJ_ID_IX_FLAG, 0, 0, 0);
+        if (res == SPIFFS_OK) { // ignore other errors
+          // got a data page also, assume lu corruption only, rewrite to new page
+          spiffs_page_ix new_pix;
+          res = spiffs_rewrite_page(fs, cur_pix, p_hdr, &new_pix);
+          SPIFFS_CHECK_DBG("LU: FIXUP: ix page with data not found elsewhere, rewriting %04x to new page %04x\n", cur_pix, new_pix);
+          SPIFFS_CHECK_RES(res);
+          *reload_lu = 1;
+          if (fs->check_cb_f) fs->check_cb_f(SPIFFS_CHECK_LOOKUP, SPIFFS_CHECK_FIX_LOOKUP, p_hdr->obj_id, p_hdr->span_ix);
+        }
+      } else {
+        SPIFFS_CHECK_RES(res);
+      }
+    }
+  }
+  if (lu_obj_id != SPIFFS_OBJ_ID_FREE && lu_obj_id != SPIFFS_OBJ_ID_DELETED) {
+    // look up entry used
+    if ((p_hdr->obj_id | SPIFFS_OBJ_ID_IX_FLAG) != (lu_obj_id | SPIFFS_OBJ_ID_IX_FLAG)) {
+      SPIFFS_CHECK_DBG("LU: pix %04x differ in obj_id lu:%04x ph:%04x\n", cur_pix, lu_obj_id, p_hdr->obj_id);
+      delete_page = 1;
+      if ((p_hdr->flags & SPIFFS_PH_FLAG_DELET) == 0 ||
+          (p_hdr->flags & SPIFFS_PH_FLAG_FINAL) ||
+          (p_hdr->flags & (SPIFFS_PH_FLAG_INDEX | SPIFFS_PH_FLAG_IXDELE)) == 0) {
+        // page deleted or not finalized, just remove it
+      } else {
+        if (p_hdr->flags & SPIFFS_PH_FLAG_INDEX) {
+          // if data page, check for reference to this page
+          res = spiffs_object_get_data_page_index_reference(fs, p_hdr->obj_id, p_hdr->span_ix, &ref_pix, &objix_pix);
+          if (res == SPIFFS_ERR_NOT_FOUND) {
+            // no object with this id, so remove page safely
+            res = SPIFFS_OK;
+          } else {
+            SPIFFS_CHECK_RES(res);
+            //   if found, rewrite page with object id, update index, and delete current
+            if (ref_pix == cur_pix) {
+              spiffs_page_ix new_pix;
+              res = spiffs_rewrite_page(fs, cur_pix, p_hdr, &new_pix);
+              SPIFFS_CHECK_RES(res);
+              res = spiffs_rewrite_index(fs, p_hdr->obj_id, p_hdr->span_ix, new_pix, objix_pix);
+              if (res <= _SPIFFS_ERR_CHECK_FIRST && res > _SPIFFS_ERR_CHECK_LAST) {
+                // index bad also, cannot mend this file
+                SPIFFS_CHECK_DBG("LU: FIXUP: index bad %d, cannot mend!\n", res);
+                res = spiffs_page_delete(fs, new_pix);
+                SPIFFS_CHECK_RES(res);
+                res = spiffs_delete_obj_lazy(fs, p_hdr->obj_id);
+                *reload_lu = 1;
+                if (fs->check_cb_f) fs->check_cb_f(SPIFFS_CHECK_LOOKUP, SPIFFS_CHECK_DELETE_BAD_FILE, p_hdr->obj_id, 0);
+              }
+              SPIFFS_CHECK_RES(res);
+            }
+          }
+        } else {
+          // else if index, check for other pages with both obj_id's and spanix
+          spiffs_page_ix objix_pix_lu, objix_pix_ph;
+          // see if other object index page exists for lookup obj id and span index
+          res = spiffs_obj_lu_find_id_and_span(fs, lu_obj_id | SPIFFS_OBJ_ID_IX_FLAG, p_hdr->span_ix, 0, &objix_pix_lu);
+          if (res == SPIFFS_ERR_NOT_FOUND) {
+            res = SPIFFS_OK;
+            objix_pix_lu = 0;
+          }
+          SPIFFS_CHECK_RES(res);
+          // see if other object index exists for page header obj id and span index
+          res = spiffs_obj_lu_find_id_and_span(fs, p_hdr->obj_id | SPIFFS_OBJ_ID_IX_FLAG, p_hdr->span_ix, 0, &objix_pix_ph);
+          if (res == SPIFFS_ERR_NOT_FOUND) {
+            res = SPIFFS_OK;
+            objix_pix_ph = 0;
+          }
+          SPIFFS_CHECK_RES(res);
+          //   if both obj_id's found, just delete current
+          if (objix_pix_ph == 0 || objix_pix_lu == 0) {
+            // otherwise try finding first corresponding data pages
+            spiffs_page_ix data_pix_lu, data_pix_ph;
+            // see if other data page exists for look up obj id and span index
+            res = spiffs_obj_lu_find_id_and_span(fs, lu_obj_id & ~SPIFFS_OBJ_ID_IX_FLAG, 0, 0, &data_pix_lu);
+            if (res == SPIFFS_ERR_NOT_FOUND) {
+              res = SPIFFS_OK;
+              objix_pix_lu = 0;
+            }
+            SPIFFS_CHECK_RES(res);
+            // see if other data page exists for page header obj id and span index
+            res = spiffs_obj_lu_find_id_and_span(fs, p_hdr->obj_id & ~SPIFFS_OBJ_ID_IX_FLAG, 0, 0, &data_pix_ph);
+            if (res == SPIFFS_ERR_NOT_FOUND) {
+              res = SPIFFS_OK;
+              objix_pix_ph = 0;
+            }
+            SPIFFS_CHECK_RES(res);
+
+            spiffs_page_header new_ph;
+            new_ph.flags = 0xff & ~(SPIFFS_PH_FLAG_USED | SPIFFS_PH_FLAG_INDEX | SPIFFS_PH_FLAG_FINAL);
+            new_ph.span_ix = p_hdr->span_ix;
+            spiffs_page_ix new_pix;
+            if ((objix_pix_lu && data_pix_lu && data_pix_ph && objix_pix_ph == 0) ||
+                (objix_pix_lu == 0 && data_pix_ph && objix_pix_ph == 0)) {
+              //   got a data page for page header obj id
+              //   rewrite as obj_id_ph
+              new_ph.obj_id = p_hdr->obj_id | SPIFFS_OBJ_ID_IX_FLAG;
+              res = spiffs_rewrite_page(fs, cur_pix, &new_ph, &new_pix);
+              SPIFFS_CHECK_DBG("LU: FIXUP: rewrite page %04x as %04x to pix %04x\n", cur_pix, new_ph.obj_id, new_pix);
+              if (fs->check_cb_f) fs->check_cb_f(SPIFFS_CHECK_LOOKUP, SPIFFS_CHECK_FIX_LOOKUP, p_hdr->obj_id, p_hdr->span_ix);
+              SPIFFS_CHECK_RES(res);
+              *reload_lu = 1;
+            } else if ((objix_pix_ph && data_pix_ph && data_pix_lu && objix_pix_lu == 0) ||
+                (objix_pix_ph == 0 && data_pix_lu && objix_pix_lu == 0)) {
+              //   got a data page for look up obj id
+              //   rewrite as obj_id_lu
+              new_ph.obj_id =  lu_obj_id | SPIFFS_OBJ_ID_IX_FLAG;
+              SPIFFS_CHECK_DBG("LU: FIXUP: rewrite page %04x as %04x\n", cur_pix, new_ph.obj_id);
+              if (fs->check_cb_f) fs->check_cb_f(SPIFFS_CHECK_LOOKUP, SPIFFS_CHECK_FIX_LOOKUP, p_hdr->obj_id, p_hdr->span_ix);
+              res = spiffs_rewrite_page(fs, cur_pix, &new_ph, &new_pix);
+              SPIFFS_CHECK_RES(res);
+              *reload_lu = 1;
+            } else {
+              // cannot safely do anything
+              SPIFFS_CHECK_DBG("LU: FIXUP: nothing to do, just delete\n");
+            }
+          }
+        }
+      }
+    } else if (((lu_obj_id & SPIFFS_OBJ_ID_IX_FLAG) && (p_hdr->flags & SPIFFS_PH_FLAG_INDEX)) ||
+        ((lu_obj_id & SPIFFS_OBJ_ID_IX_FLAG) == 0 && (p_hdr->flags & SPIFFS_PH_FLAG_INDEX) == 0)) {
+      SPIFFS_CHECK_DBG("LU: %04x lu/page index marking differ\n", cur_pix);
+      spiffs_page_ix data_pix, objix_pix_d;
+      // see if other data page exists for given obj id and span index
+      res = spiffs_obj_lu_find_id_and_span(fs, lu_obj_id & ~SPIFFS_OBJ_ID_IX_FLAG, p_hdr->span_ix, cur_pix, &data_pix);
+      if (res == SPIFFS_ERR_NOT_FOUND) {
+        res = SPIFFS_OK;
+        data_pix = 0;
+      }
+      SPIFFS_CHECK_RES(res);
+      // see if other object index exists for given obj id and span index
+      res = spiffs_obj_lu_find_id_and_span(fs, lu_obj_id | SPIFFS_OBJ_ID_IX_FLAG, p_hdr->span_ix, cur_pix, &objix_pix_d);
+      if (res == SPIFFS_ERR_NOT_FOUND) {
+        res = SPIFFS_OK;
+        objix_pix_d = 0;
+      }
+      SPIFFS_CHECK_RES(res);
+
+      delete_page = 1;
+      // if other data page exists and object index exists, just delete page
+      if (data_pix && objix_pix_d) {
+        SPIFFS_CHECK_DBG("LU: FIXUP: other index and data page exists, simply remove\n");
+      } else
+      // if only data page exists, make this page index
+      if (data_pix && objix_pix_d == 0) {
+        SPIFFS_CHECK_DBG("LU: FIXUP: other data page exists, make this index\n");
+        if (fs->check_cb_f) fs->check_cb_f(SPIFFS_CHECK_LOOKUP, SPIFFS_CHECK_FIX_INDEX, lu_obj_id, p_hdr->span_ix);
+        spiffs_page_header new_ph;
+        spiffs_page_ix new_pix;
+        new_ph.flags = 0xff & ~(SPIFFS_PH_FLAG_USED | SPIFFS_PH_FLAG_FINAL | SPIFFS_PH_FLAG_INDEX);
+        new_ph.obj_id = lu_obj_id | SPIFFS_OBJ_ID_IX_FLAG;
+        new_ph.span_ix = p_hdr->span_ix;
+        res = spiffs_page_allocate_data(fs, new_ph.obj_id, &new_ph, 0, 0, 0, 1, &new_pix);
+        SPIFFS_CHECK_RES(res);
+        res = spiffs_phys_cpy(fs, 0, SPIFFS_PAGE_TO_PADDR(fs, new_pix) + sizeof(spiffs_page_header),
+            SPIFFS_PAGE_TO_PADDR(fs, cur_pix) + sizeof(spiffs_page_header),
+            SPIFFS_CFG_LOG_PAGE_SZ(fs) - sizeof(spiffs_page_header));
+        SPIFFS_CHECK_RES(res);
+      } else
+      // if only index exists, make data page
+      if (data_pix == 0 && objix_pix_d) {
+        SPIFFS_CHECK_DBG("LU: FIXUP: other index page exists, make this data\n");
+        if (fs->check_cb_f) fs->check_cb_f(SPIFFS_CHECK_LOOKUP, SPIFFS_CHECK_FIX_LOOKUP, lu_obj_id, p_hdr->span_ix);
+        spiffs_page_header new_ph;
+        spiffs_page_ix new_pix;
+        new_ph.flags = 0xff & ~(SPIFFS_PH_FLAG_USED | SPIFFS_PH_FLAG_FINAL);
+        new_ph.obj_id = lu_obj_id & ~SPIFFS_OBJ_ID_IX_FLAG;
+        new_ph.span_ix = p_hdr->span_ix;
+        res = spiffs_page_allocate_data(fs, new_ph.obj_id, &new_ph, 0, 0, 0, 1, &new_pix);
+        SPIFFS_CHECK_RES(res);
+        res = spiffs_phys_cpy(fs, 0, SPIFFS_PAGE_TO_PADDR(fs, new_pix) + sizeof(spiffs_page_header),
+            SPIFFS_PAGE_TO_PADDR(fs, cur_pix) + sizeof(spiffs_page_header),
+            SPIFFS_CFG_LOG_PAGE_SZ(fs) - sizeof(spiffs_page_header));
+        SPIFFS_CHECK_RES(res);
+      } else {
+        // if nothing exists, we cannot safely make a decision - delete
+      }
+    }
+    else if ((p_hdr->flags & SPIFFS_PH_FLAG_DELET) == 0) {
+      SPIFFS_CHECK_DBG("LU: pix %04x busy in lu but deleted on page\n", cur_pix);
+      delete_page = 1;
+    } else if ((p_hdr->flags & SPIFFS_PH_FLAG_FINAL)) {
+      SPIFFS_CHECK_DBG("LU: pix %04x busy but not final\n", cur_pix);
+      // page can be removed if not referenced by object index
+      *reload_lu = 1;
+      res = spiffs_object_get_data_page_index_reference(fs, lu_obj_id, p_hdr->span_ix, &ref_pix, &objix_pix);
+      if (res == SPIFFS_ERR_NOT_FOUND) {
+        // no object with this id, so remove page safely
+        res = SPIFFS_OK;
+        delete_page = 1;
+      } else {
+        SPIFFS_CHECK_RES(res);
+        if (ref_pix != cur_pix) {
+          SPIFFS_CHECK_DBG("LU: FIXUP: other finalized page is referred, just delete\n");
+          delete_page = 1;
+        } else {
+          // page referenced by object index but not final
+          // just finalize
+          SPIFFS_CHECK_DBG("LU: FIXUP: unfinalized page is referred, finalizing\n");
+          if (fs->check_cb_f) fs->check_cb_f(SPIFFS_CHECK_LOOKUP, SPIFFS_CHECK_FIX_LOOKUP, p_hdr->obj_id, p_hdr->span_ix);
+          u8_t flags = 0xff & ~SPIFFS_PH_FLAG_FINAL;
+          res = _spiffs_wr(fs, SPIFFS_OP_T_OBJ_DA | SPIFFS_OP_C_UPDT,
+              0, SPIFFS_PAGE_TO_PADDR(fs, cur_pix) + offsetof(spiffs_page_header, flags),
+              sizeof(u8_t), (u8_t*)&flags);
+        }
+      }
+    }
+  }
+
+  if (delete_page) {
+    SPIFFS_CHECK_DBG("LU: FIXUP: deleting page %04x\n", cur_pix);
+    if (fs->check_cb_f) fs->check_cb_f(SPIFFS_CHECK_LOOKUP, SPIFFS_CHECK_DELETE_PAGE, cur_pix, 0);
+    res = spiffs_page_delete(fs, cur_pix);
+    SPIFFS_CHECK_RES(res);
+  }
+
+  return res;
+}
+
+static s32_t spiffs_lookup_check_v(spiffs *fs, spiffs_obj_id obj_id, spiffs_block_ix cur_block, int cur_entry,
+    u32_t user_data, void *user_p) {
+  (void)user_data;
+  (void)user_p;
+  s32_t res = SPIFFS_OK;
+  spiffs_page_header p_hdr;
+  spiffs_page_ix cur_pix = SPIFFS_OBJ_LOOKUP_ENTRY_TO_PIX(fs, cur_block, cur_entry);
+
+  if (fs->check_cb_f) fs->check_cb_f(SPIFFS_CHECK_LOOKUP, SPIFFS_CHECK_PROGRESS,
+      (cur_block * 256)/fs->block_count, 0);
+
+  // load header
+  res = _spiffs_rd(fs, SPIFFS_OP_T_OBJ_LU2 | SPIFFS_OP_C_READ,
+      0, SPIFFS_PAGE_TO_PADDR(fs, cur_pix), sizeof(spiffs_page_header), (u8_t*)&p_hdr);
+  SPIFFS_CHECK_RES(res);
+
+  int reload_lu = 0;
+
+  res = spiffs_lookup_check_validate(fs, obj_id, &p_hdr, cur_pix, cur_block, cur_entry, &reload_lu);
+  SPIFFS_CHECK_RES(res);
+
+  if (res == SPIFFS_OK) {
+    return reload_lu ? SPIFFS_VIS_COUNTINUE_RELOAD : SPIFFS_VIS_COUNTINUE;
+  }
+  return res;
+}
+
+
+// Scans all object look up. For each entry, corresponding page header is checked for validity.
+// If an object index header page is found, this is also checked
+s32_t spiffs_lookup_consistency_check(spiffs *fs, u8_t check_all_objects) {
+  (void)check_all_objects;
+  s32_t res = SPIFFS_OK;
+
+  if (fs->check_cb_f) fs->check_cb_f(SPIFFS_CHECK_LOOKUP, SPIFFS_CHECK_PROGRESS, 0, 0);
+
+  res = spiffs_obj_lu_find_entry_visitor(fs, 0, 0, 0, 0, spiffs_lookup_check_v, 0, 0, 0, 0);
+
+  if (res == SPIFFS_VIS_END) {
+    res = SPIFFS_OK;
+  }
+
+  if (res != SPIFFS_OK) {
+    if (fs->check_cb_f) fs->check_cb_f(SPIFFS_CHECK_LOOKUP, SPIFFS_CHECK_ERROR, res, 0);
+  }
+
+  if (fs->check_cb_f) fs->check_cb_f(SPIFFS_CHECK_LOOKUP, SPIFFS_CHECK_PROGRESS, 256, 0);
+
+  return res;
+}
+
+//---------------------------------------
+// Page consistency
+
+// Scans all pages (except lu pages), reserves 4 bits in working memory for each page
+// bit 0: 0 == FREE|DELETED, 1 == USED
+// bit 1: 0 == UNREFERENCED, 1 == REFERENCED
+// bit 2: 0 == NOT_INDEX,    1 == INDEX
+// bit 3: unused
+// A consistent file system will have only pages being
+//  * x000 free, unreferenced, not index
+//  * x011 used, referenced only once, not index
+//  * x101 used, unreferenced, index
+// The working memory might not fit all pages so several scans might be needed
+static s32_t spiffs_page_consistency_check_i(spiffs *fs) {
+  const u32_t bits = 4;
+  const spiffs_page_ix pages_per_scan = SPIFFS_CFG_LOG_PAGE_SZ(fs) * 8 / bits;
+
+  s32_t res = SPIFFS_OK;
+  spiffs_page_ix pix_offset = 0;
+
+  // for each range of pages fitting into work memory
+  while (pix_offset < SPIFFS_PAGES_PER_BLOCK(fs) * fs->block_count) {
+    // set this flag to abort all checks and rescan the page range
+    u8_t restart = 0;
+    c_memset(fs->work, 0, SPIFFS_CFG_LOG_PAGE_SZ(fs));
+
+    spiffs_block_ix cur_block = 0;
+    // build consistency bitmap for id range traversing all blocks
+    while (!restart && cur_block < fs->block_count) {
+      if (fs->check_cb_f) fs->check_cb_f(SPIFFS_CHECK_PAGE, SPIFFS_CHECK_PROGRESS,
+          (pix_offset*256)/(SPIFFS_PAGES_PER_BLOCK(fs) * fs->block_count) +
+          ((((cur_block * pages_per_scan * 256)/ (SPIFFS_PAGES_PER_BLOCK(fs) * fs->block_count))) / fs->block_count),
+          0);
+
+      // traverse each page except for lookup pages
+      spiffs_page_ix cur_pix = SPIFFS_OBJ_LOOKUP_PAGES(fs) + SPIFFS_PAGES_PER_BLOCK(fs) * cur_block;
+      while (!restart && cur_pix < SPIFFS_PAGES_PER_BLOCK(fs) * (cur_block+1)) {
+        // read header
+        spiffs_page_header p_hdr;
+        res = _spiffs_rd(fs, SPIFFS_OP_T_OBJ_LU2 | SPIFFS_OP_C_READ,
+            0, SPIFFS_PAGE_TO_PADDR(fs, cur_pix), sizeof(spiffs_page_header), (u8_t*)&p_hdr);
+        SPIFFS_CHECK_RES(res);
+
+        u8_t within_range = (cur_pix >= pix_offset && cur_pix < pix_offset + pages_per_scan);
+        const u32_t pix_byte_ix = (cur_pix - pix_offset) / (8/bits);
+        const u8_t pix_bit_ix = (cur_pix & ((8/bits)-1)) * bits;
+
+        if (within_range &&
+            (p_hdr.flags & SPIFFS_PH_FLAG_DELET) && (p_hdr.flags & SPIFFS_PH_FLAG_USED) == 0) {
+          // used
+          fs->work[pix_byte_ix] |= (1<<(pix_bit_ix + 0));
+        }
+        if ((p_hdr.flags & SPIFFS_PH_FLAG_DELET) &&
+            (p_hdr.flags & SPIFFS_PH_FLAG_IXDELE) &&
+            (p_hdr.flags & (SPIFFS_PH_FLAG_INDEX | SPIFFS_PH_FLAG_USED)) == 0) {
+          // found non-deleted index
+          if (within_range) {
+            fs->work[pix_byte_ix] |= (1<<(pix_bit_ix + 2));
+          }
+
+          // load non-deleted index
+          res = _spiffs_rd(fs, SPIFFS_OP_T_OBJ_LU2 | SPIFFS_OP_C_READ,
+              0, SPIFFS_PAGE_TO_PADDR(fs, cur_pix), SPIFFS_CFG_LOG_PAGE_SZ(fs), fs->lu_work);
+          SPIFFS_CHECK_RES(res);
+
+          // traverse index for referenced pages
+          spiffs_page_ix *object_page_index;
+          spiffs_page_header *objix_p_hdr = (spiffs_page_header *)fs->lu_work;
+
+          int entries;
+          int i;
+          spiffs_span_ix data_spix_offset;
+          if (p_hdr.span_ix == 0) {
+            // object header page index
+            entries = SPIFFS_OBJ_HDR_IX_LEN(fs);
+            data_spix_offset = 0;
+            object_page_index = (spiffs_page_ix *)((u8_t *)fs->lu_work + sizeof(spiffs_page_object_ix_header));
+          } else {
+            // object page index
+            entries = SPIFFS_OBJ_IX_LEN(fs);
+            data_spix_offset = SPIFFS_OBJ_HDR_IX_LEN(fs) + SPIFFS_OBJ_IX_LEN(fs) * (p_hdr.span_ix - 1);
+            object_page_index = (spiffs_page_ix *)((u8_t *)fs->lu_work + sizeof(spiffs_page_object_ix));
+          }
+
+          // for all entries in index
+          for (i = 0; !restart && i < entries; i++) {
+            spiffs_page_ix rpix = object_page_index[i];
+            u8_t rpix_within_range = rpix >= pix_offset && rpix < pix_offset + pages_per_scan;
+
+            if ((rpix != (spiffs_page_ix)-1 && rpix > SPIFFS_MAX_PAGES(fs))
+                || (rpix_within_range && SPIFFS_IS_LOOKUP_PAGE(fs, rpix))) {
+
+              // bad reference
+              SPIFFS_CHECK_DBG("PA: pix %04x bad pix / LU referenced from page %04x\n",
+                  rpix, cur_pix);
+              // check for data page elsewhere
+              spiffs_page_ix data_pix;
+              res = spiffs_obj_lu_find_id_and_span(fs, objix_p_hdr->obj_id & ~SPIFFS_OBJ_ID_IX_FLAG,
+                  data_spix_offset + i, 0, &data_pix);
+              if (res == SPIFFS_ERR_NOT_FOUND) {
+                res = SPIFFS_OK;
+                data_pix = 0;
+              }
+              SPIFFS_CHECK_RES(res);
+              if (data_pix == 0) {
+                // if not, allocate free page
+                spiffs_page_header new_ph;
+                new_ph.flags = 0xff & ~(SPIFFS_PH_FLAG_USED | SPIFFS_PH_FLAG_FINAL);
+                new_ph.obj_id = objix_p_hdr->obj_id & ~SPIFFS_OBJ_ID_IX_FLAG;
+                new_ph.span_ix = data_spix_offset + i;
+                res = spiffs_page_allocate_data(fs, new_ph.obj_id, &new_ph, 0, 0, 0, 1, &data_pix);
+                SPIFFS_CHECK_RES(res);
+                SPIFFS_CHECK_DBG("PA: FIXUP: found no existing data page, created new @ %04x\n", data_pix);
+              }
+              // remap index
+              SPIFFS_CHECK_DBG("PA: FIXUP: rewriting index pix %04x\n", cur_pix);
+              res = spiffs_rewrite_index(fs, objix_p_hdr->obj_id | SPIFFS_OBJ_ID_IX_FLAG,
+                  data_spix_offset + i, data_pix, cur_pix);
+              if (res <= _SPIFFS_ERR_CHECK_FIRST && res > _SPIFFS_ERR_CHECK_LAST) {
+                // index bad also, cannot mend this file
+                SPIFFS_CHECK_DBG("PA: FIXUP: index bad %u, cannot mend - delete object\n", res);
+                if (fs->check_cb_f) fs->check_cb_f(SPIFFS_CHECK_PAGE, SPIFFS_CHECK_DELETE_BAD_FILE, objix_p_hdr->obj_id, 0);
+                // delete file
+                res = spiffs_page_delete(fs, cur_pix);
+              } else {
+                if (fs->check_cb_f) fs->check_cb_f(SPIFFS_CHECK_PAGE, SPIFFS_CHECK_FIX_INDEX, objix_p_hdr->obj_id, objix_p_hdr->span_ix);
+              }
+              SPIFFS_CHECK_RES(res);
+              restart = 1;
+
+            } else if (rpix_within_range) {
+
+              // valid reference
+              // read referenced page header
+              spiffs_page_header rp_hdr;
+              res = _spiffs_rd(fs, SPIFFS_OP_T_OBJ_LU2 | SPIFFS_OP_C_READ,
+                  0, SPIFFS_PAGE_TO_PADDR(fs, rpix), sizeof(spiffs_page_header), (u8_t*)&rp_hdr);
+              SPIFFS_CHECK_RES(res);
+
+              // cross reference page header check
+              if (rp_hdr.obj_id != (p_hdr.obj_id & ~SPIFFS_OBJ_ID_IX_FLAG) ||
+                  rp_hdr.span_ix != data_spix_offset + i ||
+                  (rp_hdr.flags & (SPIFFS_PH_FLAG_DELET | SPIFFS_PH_FLAG_INDEX | SPIFFS_PH_FLAG_USED)) !=
+                      (SPIFFS_PH_FLAG_DELET | SPIFFS_PH_FLAG_INDEX)) {
+               SPIFFS_CHECK_DBG("PA: pix %04x has inconsistent page header ix id/span:%04x/%04x, ref id/span:%04x/%04x flags:%02x\n",
+                    rpix, p_hdr.obj_id & ~SPIFFS_OBJ_ID_IX_FLAG, data_spix_offset + i,
+                    rp_hdr.obj_id, rp_hdr.span_ix, rp_hdr.flags);
+               // try finding correct page
+               spiffs_page_ix data_pix;
+               res = spiffs_obj_lu_find_id_and_span(fs, p_hdr.obj_id & ~SPIFFS_OBJ_ID_IX_FLAG,
+                   data_spix_offset + i, rpix, &data_pix);
+               if (res == SPIFFS_ERR_NOT_FOUND) {
+                 res = SPIFFS_OK;
+                 data_pix = 0;
+               }
+               SPIFFS_CHECK_RES(res);
+               if (data_pix == 0) {
+                 // not found, this index is badly borked
+                 SPIFFS_CHECK_DBG("PA: FIXUP: index bad, delete object id %04x\n", p_hdr.obj_id);
+                 if (fs->check_cb_f) fs->check_cb_f(SPIFFS_CHECK_PAGE, SPIFFS_CHECK_DELETE_BAD_FILE, p_hdr.obj_id, 0);
+                 res = spiffs_delete_obj_lazy(fs, p_hdr.obj_id);
+                 SPIFFS_CHECK_RES(res);
+                 break;
+               } else {
+                 // found it, so rewrite index
+                 SPIFFS_CHECK_DBG("PA: FIXUP: found correct data pix %04x, rewrite ix pix %04x id %04x\n",
+                     data_pix, cur_pix, p_hdr.obj_id);
+                 res = spiffs_rewrite_index(fs, p_hdr.obj_id, data_spix_offset + i, data_pix, cur_pix);
+                 if (res <= _SPIFFS_ERR_CHECK_FIRST && res > _SPIFFS_ERR_CHECK_LAST) {
+                   // index bad also, cannot mend this file
+                   SPIFFS_CHECK_DBG("PA: FIXUP: index bad %d, cannot mend!\n", res);
+                   if (fs->check_cb_f) fs->check_cb_f(SPIFFS_CHECK_PAGE, SPIFFS_CHECK_DELETE_BAD_FILE, p_hdr.obj_id, 0);
+                   res = spiffs_delete_obj_lazy(fs, p_hdr.obj_id);
+                 } else {
+                   if (fs->check_cb_f) fs->check_cb_f(SPIFFS_CHECK_PAGE, SPIFFS_CHECK_FIX_INDEX, p_hdr.obj_id, p_hdr.span_ix);
+                 }
+                 SPIFFS_CHECK_RES(res);
+                 restart = 1;
+               }
+              }
+              else {
+                // mark rpix as referenced
+                const u32_t rpix_byte_ix = (rpix - pix_offset) / (8/bits);
+                const u8_t rpix_bit_ix = (rpix & ((8/bits)-1)) * bits;
+                if (fs->work[rpix_byte_ix] & (1<<(rpix_bit_ix + 1))) {
+                  SPIFFS_CHECK_DBG("PA: pix %04x multiple referenced from page %04x\n",
+                      rpix, cur_pix);
+                  // Here, we should have fixed all broken references - getting this means there
+                  // must be multiple files with same object id. Only solution is to delete
+                  // the object which is referring to this page
+                  SPIFFS_CHECK_DBG("PA: FIXUP: removing object %04x and page %04x\n",
+                      p_hdr.obj_id, cur_pix);
+                  if (fs->check_cb_f) fs->check_cb_f(SPIFFS_CHECK_PAGE, SPIFFS_CHECK_DELETE_BAD_FILE, p_hdr.obj_id, 0);
+                  res = spiffs_delete_obj_lazy(fs, p_hdr.obj_id);
+                  SPIFFS_CHECK_RES(res);
+                  // extra precaution, delete this page also
+                  res = spiffs_page_delete(fs, cur_pix);
+                  SPIFFS_CHECK_RES(res);
+                  restart = 1;
+                }
+                fs->work[rpix_byte_ix] |= (1<<(rpix_bit_ix + 1));
+              }
+            }
+          } // for all index entries
+        } // found index
+
+        // next page
+        cur_pix++;
+      }
+      // next block
+      cur_block++;
+    }
+    // check consistency bitmap
+    if (!restart) {
+      spiffs_page_ix objix_pix;
+      spiffs_page_ix rpix;
+
+      u32_t byte_ix;
+      u8_t bit_ix;
+      for (byte_ix = 0; !restart && byte_ix < SPIFFS_CFG_LOG_PAGE_SZ(fs); byte_ix++) {
+        for (bit_ix = 0; !restart && bit_ix < 8/bits; bit_ix ++) {
+          u8_t bitmask = (fs->work[byte_ix] >> (bit_ix * bits)) & 0x7;
+          spiffs_page_ix cur_pix = pix_offset + byte_ix * (8/bits) + bit_ix;
+
+          // 000 ok - free, unreferenced, not index
+
+          if (bitmask == 0x1) {
+
+            // 001
+            SPIFFS_CHECK_DBG("PA: pix %04x USED, UNREFERENCED, not index\n", cur_pix);
+
+            u8_t rewrite_ix_to_this = 0;
+            u8_t delete_page = 0;
+            // check corresponding object index entry
+            spiffs_page_header p_hdr;
+            res = _spiffs_rd(fs, SPIFFS_OP_T_OBJ_LU2 | SPIFFS_OP_C_READ,
+                0, SPIFFS_PAGE_TO_PADDR(fs, cur_pix), sizeof(spiffs_page_header), (u8_t*)&p_hdr);
+            SPIFFS_CHECK_RES(res);
+
+            res = spiffs_object_get_data_page_index_reference(fs, p_hdr.obj_id, p_hdr.span_ix,
+                &rpix, &objix_pix);
+            if (res == SPIFFS_OK) {
+              if (((rpix == (spiffs_page_ix)-1 || rpix > SPIFFS_MAX_PAGES(fs)) || (SPIFFS_IS_LOOKUP_PAGE(fs, rpix)))) {
+                // pointing to a bad page altogether, rewrite index to this
+                rewrite_ix_to_this = 1;
+                SPIFFS_CHECK_DBG("PA: corresponding ref is bad: %04x, rewrite to this %04x\n", rpix, cur_pix);
+              } else {
+                // pointing to something else, check what
+                spiffs_page_header rp_hdr;
+                res = _spiffs_rd(fs, SPIFFS_OP_T_OBJ_LU2 | SPIFFS_OP_C_READ,
+                    0, SPIFFS_PAGE_TO_PADDR(fs, rpix), sizeof(spiffs_page_header), (u8_t*)&rp_hdr);
+                SPIFFS_CHECK_RES(res);
+                if (((p_hdr.obj_id & ~SPIFFS_OBJ_ID_IX_FLAG) == rp_hdr.obj_id) &&
+                    ((rp_hdr.flags & (SPIFFS_PH_FLAG_INDEX | SPIFFS_PH_FLAG_DELET | SPIFFS_PH_FLAG_USED | SPIFFS_PH_FLAG_FINAL)) ==
+                        (SPIFFS_PH_FLAG_INDEX | SPIFFS_PH_FLAG_DELET))) {
+                  // pointing to something else valid, just delete this page then
+                  SPIFFS_CHECK_DBG("PA: corresponding ref is good but different: %04x, delete this %04x\n", rpix, cur_pix);
+                  delete_page = 1;
+                } else {
+                  // pointing to something weird, update index to point to this page instead
+                  if (rpix != cur_pix) {
+                    SPIFFS_CHECK_DBG("PA: corresponding ref is weird: %04x %s%s%s%s, rewrite this %04x\n", rpix,
+                        (rp_hdr.flags & SPIFFS_PH_FLAG_INDEX) ? "" : "INDEX ",
+                            (rp_hdr.flags & SPIFFS_PH_FLAG_DELET) ? "" : "DELETED ",
+                                (rp_hdr.flags & SPIFFS_PH_FLAG_USED) ? "NOTUSED " : "",
+                                    (rp_hdr.flags & SPIFFS_PH_FLAG_FINAL) ? "NOTFINAL " : "",
+                        cur_pix);
+                    rewrite_ix_to_this = 1;
+                  } else {
+                    // should not happen, destined for fubar
+                  }
+                }
+              }
+            } else if (res == SPIFFS_ERR_NOT_FOUND) {
+              SPIFFS_CHECK_DBG("PA: corresponding ref not found, delete %04x\n", cur_pix);
+              delete_page = 1;
+              res = SPIFFS_OK;
+            }
+
+            if (rewrite_ix_to_this) {
+              // if pointing to invalid page, redirect index to this page
+              SPIFFS_CHECK_DBG("PA: FIXUP: rewrite index id %04x data spix %04x to point to this pix: %04x\n",
+                  p_hdr.obj_id, p_hdr.span_ix, cur_pix);
+              res = spiffs_rewrite_index(fs, p_hdr.obj_id, p_hdr.span_ix, cur_pix, objix_pix);
+              if (res <= _SPIFFS_ERR_CHECK_FIRST && res > _SPIFFS_ERR_CHECK_LAST) {
+                // index bad also, cannot mend this file
+                SPIFFS_CHECK_DBG("PA: FIXUP: index bad %u, cannot mend!\n", res);
+                if (fs->check_cb_f) fs->check_cb_f(SPIFFS_CHECK_PAGE, SPIFFS_CHECK_DELETE_BAD_FILE, p_hdr.obj_id, 0);
+                res = spiffs_page_delete(fs, cur_pix);
+                SPIFFS_CHECK_RES(res);
+                res = spiffs_delete_obj_lazy(fs, p_hdr.obj_id);
+              } else {
+                if (fs->check_cb_f) fs->check_cb_f(SPIFFS_CHECK_PAGE, SPIFFS_CHECK_FIX_INDEX, p_hdr.obj_id, p_hdr.span_ix);
+              }
+              SPIFFS_CHECK_RES(res);
+              restart = 1;
+              continue;
+            } else if (delete_page) {
+              SPIFFS_CHECK_DBG("PA: FIXUP: deleting page %04x\n", cur_pix);
+              if (fs->check_cb_f) fs->check_cb_f(SPIFFS_CHECK_PAGE, SPIFFS_CHECK_DELETE_PAGE, cur_pix, 0);
+              res = spiffs_page_delete(fs, cur_pix);
+            }
+            SPIFFS_CHECK_RES(res);
+          }
+          if (bitmask == 0x2) {
+
+            // 010
+            SPIFFS_CHECK_DBG("PA: pix %04x FREE, REFERENCED, not index\n", cur_pix);
+
+            // no op, this should be taken care of when checking valid references
+          }
+
+          // 011 ok - busy, referenced, not index
+
+          if (bitmask == 0x4) {
+
+            // 100
+            SPIFFS_CHECK_DBG("PA: pix %04x FREE, unreferenced, INDEX\n", cur_pix);
+
+            // this should never happen, major fubar
+          }
+
+          // 101 ok - busy, unreferenced, index
+
+          if (bitmask == 0x6) {
+
+            // 110
+            SPIFFS_CHECK_DBG("PA: pix %04x FREE, REFERENCED, INDEX\n", cur_pix);
+
+            // no op, this should be taken care of when checking valid references
+          }
+          if (bitmask == 0x7) {
+
+            // 111
+            SPIFFS_CHECK_DBG("PA: pix %04x USED, REFERENCED, INDEX\n", cur_pix);
+
+            // no op, this should be taken care of when checking valid references
+          }
+        }
+      }
+    }
+    // next page range
+    if (!restart) {
+      pix_offset += pages_per_scan;
+    }
+  } // while page range not reached end
+  return res;
+}
+
+// Checks consistency amongst all pages and fixes irregularities
+s32_t spiffs_page_consistency_check(spiffs *fs) {
+  if (fs->check_cb_f) fs->check_cb_f(SPIFFS_CHECK_PAGE, SPIFFS_CHECK_PROGRESS, 0, 0);
+  s32_t res = spiffs_page_consistency_check_i(fs);
+  if (res != SPIFFS_OK) {
+    if (fs->check_cb_f) fs->check_cb_f(SPIFFS_CHECK_PAGE, SPIFFS_CHECK_ERROR, res, 0);
+  }
+  if (fs->check_cb_f) fs->check_cb_f(SPIFFS_CHECK_PAGE, SPIFFS_CHECK_PROGRESS, 256, 0);
+  return res;
+}
+
+//---------------------------------------
+// Object index consistency
+
+// searches for given object id in temporary object id index,
+// returns the index or -1
+static int spiffs_object_index_search(spiffs *fs, spiffs_obj_id obj_id) {
+  u32_t i;
+  spiffs_obj_id *obj_table = (spiffs_obj_id *)fs->work;
+  obj_id &= ~SPIFFS_OBJ_ID_IX_FLAG;
+  for (i = 0; i < SPIFFS_CFG_LOG_PAGE_SZ(fs) / sizeof(spiffs_obj_id); i++) {
+    if ((obj_table[i] & ~SPIFFS_OBJ_ID_IX_FLAG) == obj_id) {
+      return i;
+    }
+  }
+  return -1;
+}
+
+static s32_t spiffs_object_index_consistency_check_v(spiffs *fs, spiffs_obj_id obj_id, spiffs_block_ix cur_block,
+    int cur_entry, u32_t user_data, void *user_p) {
+  (void)user_data;
+  s32_t res_c = SPIFFS_VIS_COUNTINUE;
+  s32_t res = SPIFFS_OK;
+  u32_t *log_ix = (u32_t *)user_p;
+  spiffs_obj_id *obj_table = (spiffs_obj_id *)fs->work;
+
+  if (fs->check_cb_f) fs->check_cb_f(SPIFFS_CHECK_INDEX, SPIFFS_CHECK_PROGRESS,
+      (cur_block * 256)/fs->block_count, 0);
+
+  if (obj_id != SPIFFS_OBJ_ID_FREE && obj_id != SPIFFS_OBJ_ID_DELETED && (obj_id & SPIFFS_OBJ_ID_IX_FLAG)) {
+    spiffs_page_header p_hdr;
+    spiffs_page_ix cur_pix = SPIFFS_OBJ_LOOKUP_ENTRY_TO_PIX(fs, cur_block, cur_entry);
+
+    // load header
+    res = _spiffs_rd(fs, SPIFFS_OP_T_OBJ_LU2 | SPIFFS_OP_C_READ,
+        0, SPIFFS_PAGE_TO_PADDR(fs, cur_pix), sizeof(spiffs_page_header), (u8_t*)&p_hdr);
+    SPIFFS_CHECK_RES(res);
+
+    if (p_hdr.span_ix == 0 &&
+        (p_hdr.flags & (SPIFFS_PH_FLAG_INDEX | SPIFFS_PH_FLAG_FINAL | SPIFFS_PH_FLAG_DELET | SPIFFS_PH_FLAG_IXDELE)) ==
+        (SPIFFS_PH_FLAG_DELET)) {
+      SPIFFS_CHECK_DBG("IX: pix %04x, obj id:%04x spix:%04x header not fully deleted - deleting\n",
+          cur_pix, obj_id, p_hdr.span_ix);
+      if (fs->check_cb_f) fs->check_cb_f(SPIFFS_CHECK_INDEX, SPIFFS_CHECK_DELETE_PAGE, cur_pix, obj_id);
+      res = spiffs_page_delete(fs, cur_pix);
+      SPIFFS_CHECK_RES(res);
+      return res_c;
+    }
+
+    if ((p_hdr.flags & (SPIFFS_PH_FLAG_INDEX | SPIFFS_PH_FLAG_FINAL | SPIFFS_PH_FLAG_DELET | SPIFFS_PH_FLAG_IXDELE)) ==
+        (SPIFFS_PH_FLAG_DELET | SPIFFS_PH_FLAG_IXDELE)) {
+      return res_c;
+    }
+
+    if (p_hdr.span_ix == 0) {
+      // objix header page, register objid as reachable
+      int r = spiffs_object_index_search(fs, obj_id);
+      if (r == -1) {
+        // not registered, do it
+        obj_table[*log_ix] = obj_id & ~SPIFFS_OBJ_ID_IX_FLAG;
+        (*log_ix)++;
+        if (*log_ix >= SPIFFS_CFG_LOG_PAGE_SZ(fs) / sizeof(spiffs_obj_id)) {
+          *log_ix = 0;
+        }
+      }
+    } else { // span index
+      // objix page, see if header can be found
+      int r = spiffs_object_index_search(fs, obj_id);
+      u8_t delete = 0;
+      if (r == -1) {
+        // not in temporary index, try finding it
+        spiffs_page_ix objix_hdr_pix;
+        res = spiffs_obj_lu_find_id_and_span(fs, obj_id | SPIFFS_OBJ_ID_IX_FLAG, 0, 0, &objix_hdr_pix);
+        res_c = SPIFFS_VIS_COUNTINUE_RELOAD;
+        if (res == SPIFFS_OK) {
+          // found, register as reachable
+          obj_table[*log_ix] = obj_id & ~SPIFFS_OBJ_ID_IX_FLAG;
+        } else if (res == SPIFFS_ERR_NOT_FOUND) {
+          // not found, register as unreachable
+          delete = 1;
+          obj_table[*log_ix] = obj_id | SPIFFS_OBJ_ID_IX_FLAG;
+        } else {
+          SPIFFS_CHECK_RES(res);
+        }
+        (*log_ix)++;
+        if (*log_ix >= SPIFFS_CFG_LOG_PAGE_SZ(fs) / sizeof(spiffs_obj_id)) {
+          *log_ix = 0;
+        }
+      } else {
+        // in temporary index, check reachable flag
+        if ((obj_table[r] & SPIFFS_OBJ_ID_IX_FLAG)) {
+          // registered as unreachable
+          delete = 1;
+        }
+      }
+
+      if (delete) {
+        SPIFFS_CHECK_DBG("IX: FIXUP: pix %04x, obj id:%04x spix:%04x is orphan index - deleting\n",
+            cur_pix, obj_id, p_hdr.span_ix);
+        if (fs->check_cb_f) fs->check_cb_f(SPIFFS_CHECK_INDEX, SPIFFS_CHECK_DELETE_ORPHANED_INDEX, cur_pix, obj_id);
+        res = spiffs_page_delete(fs, cur_pix);
+        SPIFFS_CHECK_RES(res);
+      }
+    } // span index
+  } // valid object index id
+
+  return res_c;
+}
+
+// Removes orphaned and partially deleted index pages.
+// Scans for index pages. When an index page is found, corresponding index header is searched for.
+// If no such page exists, the index page cannot be reached as no index header exists and must be
+// deleted.
+s32_t spiffs_object_index_consistency_check(spiffs *fs) {
+  s32_t res = SPIFFS_OK;
+  // impl note:
+  // fs->work is used for a temporary object index memory, listing found object ids and
+  // indicating whether they can be reached or not. Acting as a fifo if object ids cannot fit.
+  // In the temporary object index memory, SPIFFS_OBJ_ID_IX_FLAG bit is used to indicate
+  // a reachable/unreachable object id.
+  c_memset(fs->work, 0, SPIFFS_CFG_LOG_PAGE_SZ(fs));
+  u32_t obj_id_log_ix = 0;
+  if (fs->check_cb_f) fs->check_cb_f(SPIFFS_CHECK_INDEX, SPIFFS_CHECK_PROGRESS, 0, 0);
+  res = spiffs_obj_lu_find_entry_visitor(fs, 0, 0, 0, 0, spiffs_object_index_consistency_check_v, 0, &obj_id_log_ix,
+      0, 0);
+  if (res == SPIFFS_VIS_END) {
+    res = SPIFFS_OK;
+  }
+  if (res != SPIFFS_OK) {
+    if (fs->check_cb_f) fs->check_cb_f(SPIFFS_CHECK_INDEX, SPIFFS_CHECK_ERROR, res, 0);
+  }
+  if (fs->check_cb_f) fs->check_cb_f(SPIFFS_CHECK_INDEX, SPIFFS_CHECK_PROGRESS, 256, 0);
+  return res;
+}
+

cores/esp8266/spiffs/spiffs_config.h

@@ -0,0 +1,242 @@
+/*
+ * spiffs_config.h
+ *
+ *  Created on: Jul 3, 2013
+ *      Author: petera
+ */
+
+#ifndef SPIFFS_CONFIG_H_
+#define SPIFFS_CONFIG_H_
+
+// ----------- 8< ------------
+// Following includes are for the linux test build of spiffs
+// These may/should/must be removed/altered/replaced in your target
+// #include "params_test.h"
+//#include "c_stdio.h"
+//#include "c_stdlib.h"
+//#include "c_string.h"
+#include "mem.h"
+#include "c_types.h"
+#include "stddef.h"
+#include "osapi.h"
+#include "ets_sys.h"
+#include <user_config.h>
+// ----------- >8 ------------
+#define IRAM_ATTR __attribute__((section(".iram.text")))
+#define STORE_TYPEDEF_ATTR __attribute__((aligned(4),packed))
+#define STORE_ATTR __attribute__((aligned(4)))
+
+#define SPIFFS_CHACHE 0
+
+#define c_memcpy os_memcpy
+#define c_printf os_printf
+#define c_memset os_memset
+
+typedef signed short file_t;
+typedef long int s32_t;
+typedef long unsigned int u32_t;
+typedef int16_t s16_t;
+typedef uint16_t u16_t;
+typedef int8_t s8_t;
+typedef uint8_t u8_t;
+
+#ifndef SEEK_SET
+#define	SEEK_SET	0	/* set file offset to offset */
+#endif
+
+#ifndef SEEK_CUR
+#define	SEEK_CUR	1	/* set file offset to current plus offset */
+#endif
+
+#ifndef SEEK_END
+#define	SEEK_END	2	/* set file offset to EOF plus offset */
+#endif
+
+#ifndef EOF
+#define	EOF	(-1)
+#endif
+
+// compile time switches
+#define debugf(fmt, ...) //os_printf(fmt"\r\n", ##__VA_ARGS__)
+#define SYSTEM_ERROR(fmt, ...) //os_printf("ERROR: " fmt "\r\n", ##__VA_ARGS__)
+
+// Set generic spiffs debug output call.
+#ifndef SPIFFS_DGB
+#define SPIFFS_DBG(...) //os_printf(__VA_ARGS__)
+#endif
+// Set spiffs debug output call for garbage collecting.
+#ifndef SPIFFS_GC_DGB
+#define SPIFFS_GC_DBG(...) //os_printf(__VA_ARGS__)
+#endif
+// Set spiffs debug output call for caching.
+#ifndef SPIFFS_CACHE_DGB
+#define SPIFFS_CACHE_DBG(...) //os_printf(__VA_ARGS__)
+#endif
+// Set spiffs debug output call for system consistency checks.
+#ifndef SPIFFS_CHECK_DGB
+#define SPIFFS_CHECK_DBG(...) //os_printf(__VA_ARGS__)
+#endif
+
+// Enable/disable API functions to determine exact number of bytes
+// for filedescriptor and cache buffers. Once decided for a configuration,
+// this can be disabled to reduce flash.
+#ifndef SPIFFS_BUFFER_HELP
+#define SPIFFS_BUFFER_HELP              0
+#endif
+
+// Enables/disable memory read caching of nucleus file system operations.
+// If enabled, memory area must be provided for cache in SPIFFS_mount.
+#ifndef  SPIFFS_CACHE
+#define SPIFFS_CACHE                    1
+#endif
+#if SPIFFS_CACHE
+// Enables memory write caching for file descriptors in hydrogen
+#ifndef  SPIFFS_CACHE_WR
+#define SPIFFS_CACHE_WR                 1
+#endif
+
+// Enable/disable statistics on caching. Debug/test purpose only.
+#ifndef  SPIFFS_CACHE_STATS
+#define SPIFFS_CACHE_STATS              0
+#endif
+#endif
+
+// Always check header of each accessed page to ensure consistent state.
+// If enabled it will increase number of reads, will increase flash.
+#ifndef SPIFFS_PAGE_CHECK
+#define SPIFFS_PAGE_CHECK               1
+#endif
+
+// Define maximum number of gc runs to perform to reach desired free pages.
+#ifndef SPIFFS_GC_MAX_RUNS
+#define SPIFFS_GC_MAX_RUNS              3
+#endif
+
+// Enable/disable statistics on gc. Debug/test purpose only.
+#ifndef SPIFFS_GC_STATS
+#define SPIFFS_GC_STATS                 0
+#endif
+
+// Garbage collecting examines all pages in a block which and sums up
+// to a block score. Deleted pages normally gives positive score and
+// used pages normally gives a negative score (as these must be moved).
+// To have a fair wear-leveling, the erase age is also included in score,
+// whose factor normally is the most positive.
+// The larger the score, the more likely it is that the block will
+// picked for garbage collection.
+
+// Garbage collecting heuristics - weight used for deleted pages.
+#ifndef SPIFFS_GC_HEUR_W_DELET
+#define SPIFFS_GC_HEUR_W_DELET          (5)
+#endif
+// Garbage collecting heuristics - weight used for used pages.
+#ifndef SPIFFS_GC_HEUR_W_USED
+#define SPIFFS_GC_HEUR_W_USED           (-1)
+#endif
+// Garbage collecting heuristics - weight used for time between
+// last erased and erase of this block.
+#ifndef SPIFFS_GC_HEUR_W_ERASE_AGE
+#define SPIFFS_GC_HEUR_W_ERASE_AGE      (50)
+#endif
+
+// Object name maximum length.
+#ifndef SPIFFS_OBJ_NAME_LEN
+#define SPIFFS_OBJ_NAME_LEN             (32)
+#endif
+
+// Size of buffer allocated on stack used when copying data.
+// Lower value generates more read/writes. No meaning having it bigger
+// than logical page size.
+#ifndef SPIFFS_COPY_BUFFER_STACK
+#define SPIFFS_COPY_BUFFER_STACK        (64)
+#endif
+
+// SPIFFS_LOCK and SPIFFS_UNLOCK protects spiffs from reentrancy on api level
+// These should be defined on a multithreaded system
+
+// define this to entering a mutex if you're running on a multithreaded system
+#ifndef SPIFFS_LOCK
+#define SPIFFS_LOCK(fs)
+#endif
+// define this to exiting a mutex if you're running on a multithreaded system
+#ifndef SPIFFS_UNLOCK
+#define SPIFFS_UNLOCK(fs)
+#endif
+
+
+// Enable if only one spiffs instance with constant configuration will exist
+// on the target. This will reduce calculations, flash and memory accesses.
+// Parts of configuration must be defined below instead of at time of mount.
+#ifndef SPIFFS_SINGLETON
+#define SPIFFS_SINGLETON 0
+#endif
+
+#if SPIFFS_SINGLETON
+// Instead of giving parameters in config struct, singleton build must
+// give parameters in defines below.
+#ifndef SPIFFS_CFG_PHYS_SZ
+#define SPIFFS_CFG_PHYS_SZ(ignore)        (1024*1024*2)
+#endif
+#ifndef SPIFFS_CFG_PHYS_ERASE_SZ
+#define SPIFFS_CFG_PHYS_ERASE_SZ(ignore)  (65536)
+#endif
+#ifndef SPIFFS_CFG_PHYS_ADDR
+#define SPIFFS_CFG_PHYS_ADDR(ignore)      (0)
+#endif
+#ifndef SPIFFS_CFG_LOG_PAGE_SZ
+#define SPIFFS_CFG_LOG_PAGE_SZ(ignore)    (256)
+#endif
+#ifndef SPIFFS_CFG_LOG_BLOCK_SZ
+#define SPIFFS_CFG_LOG_BLOCK_SZ(ignore)   (65536)
+#endif
+#endif
+
+// Set SPFIFS_TEST_VISUALISATION to non-zero to enable SPIFFS_vis function
+// in the api. This function will visualize all filesystem using given printf
+// function.
+#ifndef SPIFFS_TEST_VISUALISATION
+#define SPIFFS_TEST_VISUALISATION         1
+#endif
+#if SPIFFS_TEST_VISUALISATION
+#ifndef spiffs_printf
+#define spiffs_printf(...)                c_printf(__VA_ARGS__)
+#endif
+// spiffs_printf argument for a free page
+#ifndef SPIFFS_TEST_VIS_FREE_STR
+#define SPIFFS_TEST_VIS_FREE_STR          "_"
+#endif
+// spiffs_printf argument for a deleted page
+#ifndef SPIFFS_TEST_VIS_DELE_STR
+#define SPIFFS_TEST_VIS_DELE_STR          "/"
+#endif
+// spiffs_printf argument for an index page for given object id
+#ifndef SPIFFS_TEST_VIS_INDX_STR
+#define SPIFFS_TEST_VIS_INDX_STR(id)      "i"
+#endif
+// spiffs_printf argument for a data page for given object id
+#ifndef SPIFFS_TEST_VIS_DATA_STR
+#define SPIFFS_TEST_VIS_DATA_STR(id)      "d"
+#endif
+#endif
+
+// Types depending on configuration such as the amount of flash bytes
+// given to spiffs file system in total (spiffs_file_system_size),
+// the logical block size (log_block_size), and the logical page size
+// (log_page_size)
+
+// Block index type. Make sure the size of this type can hold
+// the highest number of all blocks - i.e. spiffs_file_system_size / log_block_size
+typedef u16_t spiffs_block_ix;
+// Page index type. Make sure the size of this type can hold
+// the highest page number of all pages - i.e. spiffs_file_system_size / log_page_size
+typedef u16_t spiffs_page_ix;
+// Object id type - most significant bit is reserved for index flag. Make sure the
+// size of this type can hold the highest object id on a full system,
+// i.e. 2 + (spiffs_file_system_size / (2*log_page_size))*2
+typedef u16_t spiffs_obj_id;
+// Object span index type. Make sure the size of this type can
+// hold the largest possible span index on the system -
+// i.e. (spiffs_file_system_size / log_page_size) - 1
+typedef u16_t spiffs_span_ix;
+
+#endif /* SPIFFS_CONFIG_H_ */

cores/esp8266/spiffs/spiffs_flashmem.c

@@ -0,0 +1,223 @@
+#include "flashmem.h"
+
+// Based on NodeMCU platform_flash
+// https://github.com/nodemcu/nodemcu-firmware
+
+extern uint32_t _SPIFFS_start;
+
+uint32_t flashmem_write( const void *from, uint32_t toaddr, uint32_t size )
+{
+  uint32_t temp, rest, ssize = size;
+  unsigned i;
+  char tmpdata[ INTERNAL_FLASH_WRITE_UNIT_SIZE ];
+  const uint8_t *pfrom = ( const uint8_t* )from;
+  const uint32_t blksize = INTERNAL_FLASH_WRITE_UNIT_SIZE;
+  const uint32_t blkmask = INTERNAL_FLASH_WRITE_UNIT_SIZE - 1;
+
+  // Align the start
+  if( toaddr & blkmask )
+  {
+    rest = toaddr & blkmask;
+    temp = toaddr & ~blkmask; // this is the actual aligned address
+    // c_memcpy( tmpdata, ( const void* )temp, blksize );
+    flashmem_read_internal( tmpdata, temp, blksize );
+    for( i = rest; size && ( i < blksize ); i ++, size --, pfrom ++ )
+      tmpdata[ i ] = *pfrom;
+    flashmem_write_internal( tmpdata, temp, blksize );
+    if( size == 0 )
+      return ssize;
+    toaddr = temp + blksize;
+  }
+  // The start address is now a multiple of blksize
+  // Compute how many bytes we can write as multiples of blksize
+  rest = size & blkmask;
+  temp = size & ~blkmask;
+  // Program the blocks now
+  if( temp )
+  {
+	flashmem_write_internal( pfrom, toaddr, temp );
+    toaddr += temp;
+    pfrom += temp;
+  }
+  // And the final part of a block if needed
+  if( rest )
+  {
+    // c_memcpy( tmpdata, ( const void* )toaddr, blksize );
+	flashmem_read_internal( tmpdata, toaddr, blksize );
+    for( i = 0; size && ( i < rest ); i ++, size --, pfrom ++ )
+      tmpdata[ i ] = *pfrom;
+    flashmem_write_internal( tmpdata, toaddr, blksize );
+  }
+  return ssize;
+}
+
+uint32_t flashmem_read( void *to, uint32_t fromaddr, uint32_t size )
+{
+  uint32_t temp, rest, ssize = size;
+  unsigned i;
+  char tmpdata[ INTERNAL_FLASH_READ_UNIT_SIZE ];
+  uint8_t *pto = ( uint8_t* )to;
+  const uint32_t blksize = INTERNAL_FLASH_READ_UNIT_SIZE;
+  const uint32_t blkmask = INTERNAL_FLASH_READ_UNIT_SIZE - 1;
+
+  // Align the start
+  if( fromaddr & blkmask )
+  {
+    rest = fromaddr & blkmask;
+    temp = fromaddr & ~blkmask; // this is the actual aligned address
+    flashmem_read_internal( tmpdata, temp, blksize );
+    for( i = rest; size && ( i < blksize ); i ++, size --, pto ++ )
+      *pto = tmpdata[ i ];
+
+    if( size == 0 )
+      return ssize;
+    fromaddr = temp + blksize;
+  }
+  // The start address is now a multiple of blksize
+  // Compute how many bytes we can read as multiples of blksize
+  rest = size & blkmask;
+  temp = size & ~blkmask;
+  // Program the blocks now
+  if( temp )
+  {
+	flashmem_read_internal( pto, fromaddr, temp );
+    fromaddr += temp;
+    pto += temp;
+  }
+  // And the final part of a block if needed
+  if( rest )
+  {
+	flashmem_read_internal( tmpdata, fromaddr, blksize );
+    for( i = 0; size && ( i < rest ); i ++, size --, pto ++ )
+      *pto = tmpdata[ i ];
+  }
+  return ssize;
+}
+
+bool flashmem_erase_sector( uint32_t sector_id )
+{
+  WRITE_PERI_REG(0x60000914, 0x73);
+  return spi_flash_erase_sector( sector_id ) == SPI_FLASH_RESULT_OK;
+}
+
+SPIFlashInfo flashmem_get_info()
+{
+    volatile SPIFlashInfo spi_flash_info STORE_ATTR;
+    spi_flash_info = *((SPIFlashInfo *)(INTERNAL_FLASH_START_ADDRESS));
+    return spi_flash_info;
+}
+
+uint8_t flashmem_get_size_type()
+{
+    return flashmem_get_info().size;
+}
+
+uint32_t flashmem_get_size_bytes()
+{
+    uint32_t flash_size = 0;
+    switch (flashmem_get_info().size)
+    {
+    case SIZE_2MBIT:
+        // 2Mbit, 256kByte
+        flash_size = 256 * 1024;
+        break;
+    case SIZE_4MBIT:
+        // 4Mbit, 512kByte
+        flash_size = 512 * 1024;
+        break;
+    case SIZE_8MBIT:
+        // 8Mbit, 1MByte
+        flash_size = 1 * 1024 * 1024;
+        break;
+    case SIZE_16MBIT:
+        // 16Mbit, 2MByte
+        flash_size = 2 * 1024 * 1024;
+        break;
+    case SIZE_32MBIT:
+        // 32Mbit, 4MByte
+        flash_size = 4 * 1024 * 1024;
+        break;
+    default:
+        // Unknown flash size, fall back mode.
+        flash_size = 512 * 1024;
+        break;
+    }
+    return flash_size;
+}
+
+uint16_t flashmem_get_size_sectors()
+{
+    return flashmem_get_size_bytes() / SPI_FLASH_SEC_SIZE;
+}
+
+// Helper function: find the flash sector in which an address resides
+// Return the sector number, as well as the start and end address of the sector
+uint32_t flashmem_find_sector( uint32_t address, uint32_t *pstart, uint32_t *pend )
+{
+  address -= INTERNAL_FLASH_START_ADDRESS;
+  // All the sectors in the flash have the same size, so just align the address
+  uint32_t sect_id = address / INTERNAL_FLASH_SECTOR_SIZE;
+
+  if( pstart )
+    *pstart = sect_id * INTERNAL_FLASH_SECTOR_SIZE + INTERNAL_FLASH_START_ADDRESS;
+  if( pend )
+    *pend = ( sect_id + 1 ) * INTERNAL_FLASH_SECTOR_SIZE + INTERNAL_FLASH_START_ADDRESS - 1;
+  return sect_id;
+}
+
+uint32_t flashmem_get_sector_of_address( uint32_t addr )
+{
+  return flashmem_find_sector( addr, NULL, NULL );
+}
+
+/////////////////////////////////////////////////////
+
+uint32_t flashmem_write_internal( const void *from, uint32_t toaddr, uint32_t size )
+{
+  toaddr -= INTERNAL_FLASH_START_ADDRESS;
+  SpiFlashOpResult r;
+  const uint32_t blkmask = INTERNAL_FLASH_WRITE_UNIT_SIZE - 1;
+  uint32_t *apbuf = NULL;
+  if( ((uint32_t)from) & blkmask ){
+    apbuf = (uint32_t *)os_malloc(size);
+    if(!apbuf)
+      return 0;
+    os_memcpy(apbuf, from, size);
+  }
+  WRITE_PERI_REG(0x60000914, 0x73);
+  r = spi_flash_write(toaddr, apbuf?(uint32 *)apbuf:(uint32 *)from, size);
+  if(apbuf)
+    os_free(apbuf);
+  if(SPI_FLASH_RESULT_OK == r)
+    return size;
+  else{
+	SYSTEM_ERROR( "ERROR in flash_write: r=%d at %08X\n", ( int )r, ( unsigned )toaddr+INTERNAL_FLASH_START_ADDRESS );
+    return 0;
+  }
+}
+
+uint32_t flashmem_read_internal( void *to, uint32_t fromaddr, uint32_t size )
+{
+  fromaddr -= INTERNAL_FLASH_START_ADDRESS;
+  SpiFlashOpResult r;
+  WRITE_PERI_REG(0x60000914, 0x73);
+  r = spi_flash_read(fromaddr, (uint32 *)to, size);
+  if(SPI_FLASH_RESULT_OK == r)
+    return size;
+  else{
+	  SYSTEM_ERROR( "ERROR in flash_read: r=%d at %08X\n", ( int )r, ( unsigned )fromaddr+INTERNAL_FLASH_START_ADDRESS );
+    return 0;
+  }
+}
+
+uint32_t flashmem_get_first_free_block_address(){
+  if ((uint32_t)&_SPIFFS_start == 0){
+	  return 0;
+  }
+	debugf("_SPIFFS_start:%08x\n", (uint32_t)&_SPIFFS_start);
+
+  // Round the total used flash size to the closest flash block address
+  uint32_t end;
+  flashmem_find_sector( (uint32_t)&_SPIFFS_start - 1, NULL, &end);
+  return end + 1;
+}

cores/esp8266/spiffs/spiffs_gc.c

@@ -0,0 +1,555 @@
+#include "spiffs.h"
+#include "spiffs_nucleus.h"
+
+// Erases a logical block and updates the erase counter.
+// If cache is enabled, all pages that might be cached in this block
+// is dropped.
+static s32_t spiffs_gc_erase_block(
+    spiffs *fs,
+    spiffs_block_ix bix) {
+  s32_t res;
+  u32_t addr = SPIFFS_BLOCK_TO_PADDR(fs, bix);
+  s32_t size = SPIFFS_CFG_LOG_BLOCK_SZ(fs);
+
+  SPIFFS_GC_DBG("gc: erase block %d\n", bix);
+
+  // here we ignore res, just try erasing the block
+  while (size > 0) {
+    SPIFFS_GC_DBG("gc: erase %08x:%08x\n", addr,  SPIFFS_CFG_PHYS_ERASE_SZ(fs));
+    (void)fs->cfg.hal_erase_f(addr, SPIFFS_CFG_PHYS_ERASE_SZ(fs));
+    addr += SPIFFS_CFG_PHYS_ERASE_SZ(fs);
+    size -= SPIFFS_CFG_PHYS_ERASE_SZ(fs);
+  }
+  fs->free_blocks++;
+
+  // register erase count for this block
+  res = _spiffs_wr(fs, SPIFFS_OP_C_WRTHRU | SPIFFS_OP_T_OBJ_LU2, 0,
+      SPIFFS_ERASE_COUNT_PADDR(fs, bix),
+      sizeof(spiffs_obj_id), (u8_t *)&fs->max_erase_count);
+  SPIFFS_CHECK_RES(res);
+
+  fs->max_erase_count++;
+  if (fs->max_erase_count == SPIFFS_OBJ_ID_IX_FLAG) {
+    fs->max_erase_count = 0;
+  }
+
+#if SPIFFS_CACHE
+  {
+    u32_t i;
+    for (i = 0; i < SPIFFS_PAGES_PER_BLOCK(fs); i++) {
+      spiffs_cache_drop_page(fs, SPIFFS_PAGE_FOR_BLOCK(fs, bix) + i);
+    }
+  }
+#endif
+  return res;
+}
+
+// Searches for blocks where all entries are deleted - if one is found,
+// the block is erased. Compared to the non-quick gc, the quick one ensures
+// that no updates are needed on existing objects on pages that are erased.
+s32_t spiffs_gc_quick(
+    spiffs *fs) {
+  s32_t res = SPIFFS_OK;
+  u32_t blocks = fs->block_count;
+  spiffs_block_ix cur_block = 0;
+  u32_t cur_block_addr = 0;
+  int cur_entry = 0;
+  spiffs_obj_id *obj_lu_buf = (spiffs_obj_id *)fs->lu_work;
+
+  SPIFFS_GC_DBG("gc_quick: running\n", cur_block);
+#if SPIFFS_GC_STATS
+  fs->stats_gc_runs++;
+#endif
+
+  int entries_per_page = (SPIFFS_CFG_LOG_PAGE_SZ(fs) / sizeof(spiffs_obj_id));
+
+  // find fully deleted blocks
+  // check each block
+  while (res == SPIFFS_OK && blocks--) {
+    u16_t deleted_pages_in_block = 0;
+
+    int obj_lookup_page = 0;
+    // check each object lookup page
+    while (res == SPIFFS_OK && obj_lookup_page < (int)SPIFFS_OBJ_LOOKUP_PAGES(fs)) {
+      int entry_offset = obj_lookup_page * entries_per_page;
+      res = _spiffs_rd(fs, SPIFFS_OP_T_OBJ_LU | SPIFFS_OP_C_READ,
+          0, cur_block_addr + SPIFFS_PAGE_TO_PADDR(fs, obj_lookup_page), SPIFFS_CFG_LOG_PAGE_SZ(fs), fs->lu_work);
+      // check each entry
+      while (res == SPIFFS_OK &&
+          cur_entry - entry_offset < entries_per_page &&
+          cur_entry < (int)(SPIFFS_PAGES_PER_BLOCK(fs)-SPIFFS_OBJ_LOOKUP_PAGES(fs))) {
+        spiffs_obj_id obj_id = obj_lu_buf[cur_entry-entry_offset];
+        if (obj_id == SPIFFS_OBJ_ID_DELETED) {
+          deleted_pages_in_block++;
+        } else if (obj_id == SPIFFS_OBJ_ID_FREE) {
+          // kill scan, go for next block
+          obj_lookup_page = SPIFFS_OBJ_LOOKUP_PAGES(fs);
+          res = 1; // kill object lu loop
+          break;
+        }  else {
+          // kill scan, go for next block
+          obj_lookup_page = SPIFFS_OBJ_LOOKUP_PAGES(fs);
+          res = 1; // kill object lu loop
+          break;
+        }
+        cur_entry++;
+      } // per entry
+      obj_lookup_page++;
+    } // per object lookup page
+    if (res == 1) res = SPIFFS_OK;
+
+    if (res == SPIFFS_OK && deleted_pages_in_block == SPIFFS_PAGES_PER_BLOCK(fs)-SPIFFS_OBJ_LOOKUP_PAGES(fs)) {
+      // found a fully deleted block
+      fs->stats_p_deleted -= deleted_pages_in_block;
+      res = spiffs_gc_erase_block(fs, cur_block);
+      return res;
+    }
+
+    cur_entry = 0;
+    cur_block++;
+    cur_block_addr += SPIFFS_CFG_LOG_BLOCK_SZ(fs);
+  } // per block
+
+  return res;
+}
+
+// Checks if garbaga collecting is necessary. If so a candidate block is found,
+// cleansed and erased
+s32_t spiffs_gc_check(
+    spiffs *fs,
+    u32_t len) {
+  s32_t res;
+  u32_t free_pages =
+      (SPIFFS_PAGES_PER_BLOCK(fs) - SPIFFS_OBJ_LOOKUP_PAGES(fs)) * fs->block_count
+      - fs->stats_p_allocated - fs->stats_p_deleted;
+  int tries = 0;
+
+  if (fs->free_blocks > 3 &&
+      len < free_pages * SPIFFS_DATA_PAGE_SIZE(fs)) {
+    return SPIFFS_OK;
+  }
+
+  //printf("gcing started  %d dirty, blocks %d free, want %d bytes\n", fs->stats_p_allocated + fs->stats_p_deleted, fs->free_blocks, len);
+
+  do {
+    SPIFFS_GC_DBG("\ngc_check #%d: run gc free_blocks:%d pfree:%d pallo:%d pdele:%d [%d] len:%d of %d\n",
+        tries,
+        fs->free_blocks, free_pages, fs->stats_p_allocated, fs->stats_p_deleted, (free_pages+fs->stats_p_allocated+fs->stats_p_deleted),
+        len, free_pages*SPIFFS_DATA_PAGE_SIZE(fs));
+
+    spiffs_block_ix *cands;
+    int count;
+    spiffs_block_ix cand;
+    res = spiffs_gc_find_candidate(fs, &cands, &count);
+    SPIFFS_CHECK_RES(res);
+    if (count == 0) {
+      SPIFFS_GC_DBG("gc_check: no candidates, return\n");
+      return res;
+    }
+#if SPIFFS_GC_STATS
+    fs->stats_gc_runs++;
+#endif
+    cand = cands[0];
+    fs->cleaning = 1;
+    //printf("gcing: cleaning block %d\n", cand);
+    res = spiffs_gc_clean(fs, cand);
+    fs->cleaning = 0;
+    if (res < 0) {
+      SPIFFS_GC_DBG("gc_check: cleaning block %d, result %d\n", cand, res);
+    } else {
+      SPIFFS_GC_DBG("gc_check: cleaning block %d, result %d\n", cand, res);
+    }
+    SPIFFS_CHECK_RES(res);
+
+    res = spiffs_gc_erase_page_stats(fs, cand);
+    SPIFFS_CHECK_RES(res);
+
+    res = spiffs_gc_erase_block(fs, cand);
+    SPIFFS_CHECK_RES(res);
+
+    free_pages =
+          (SPIFFS_PAGES_PER_BLOCK(fs) - SPIFFS_OBJ_LOOKUP_PAGES(fs)) * fs->block_count
+          - fs->stats_p_allocated - fs->stats_p_deleted;
+
+  } while (++tries < SPIFFS_GC_MAX_RUNS && (fs->free_blocks <= 2 ||
+      len > free_pages*SPIFFS_DATA_PAGE_SIZE(fs)));
+  SPIFFS_GC_DBG("gc_check: finished\n");
+
+  //printf("gcing finished %d dirty, blocks %d free, %d pages free, %d tries, res %d\n",
+  //    fs->stats_p_allocated + fs->stats_p_deleted,
+  //    fs->free_blocks, free_pages, tries, res);
+
+  return res;
+}
+
+// Updates page statistics for a block that is about to be erased
+s32_t spiffs_gc_erase_page_stats(
+    spiffs *fs,
+    spiffs_block_ix bix) {
+  s32_t res = SPIFFS_OK;
+  int obj_lookup_page = 0;
+  int entries_per_page = (SPIFFS_CFG_LOG_PAGE_SZ(fs) / sizeof(spiffs_obj_id));
+  spiffs_obj_id *obj_lu_buf = (spiffs_obj_id *)fs->lu_work;
+  int cur_entry = 0;
+  u32_t dele = 0;
+  u32_t allo = 0;
+
+  // check each object lookup page
+  while (res == SPIFFS_OK && obj_lookup_page < (int)SPIFFS_OBJ_LOOKUP_PAGES(fs)) {
+    int entry_offset = obj_lookup_page * entries_per_page;
+    res = _spiffs_rd(fs, SPIFFS_OP_T_OBJ_LU | SPIFFS_OP_C_READ,
+        0, bix * SPIFFS_CFG_LOG_BLOCK_SZ(fs) + SPIFFS_PAGE_TO_PADDR(fs, obj_lookup_page), SPIFFS_CFG_LOG_PAGE_SZ(fs), fs->lu_work);
+    // check each entry
+    while (res == SPIFFS_OK &&
+        cur_entry - entry_offset < entries_per_page && cur_entry < (int)(SPIFFS_PAGES_PER_BLOCK(fs)-SPIFFS_OBJ_LOOKUP_PAGES(fs))) {
+      spiffs_obj_id obj_id = obj_lu_buf[cur_entry-entry_offset];
+      if (obj_id == SPIFFS_OBJ_ID_FREE) {
+      } else if (obj_id == SPIFFS_OBJ_ID_DELETED) {
+        dele++;
+      } else {
+        allo++;
+      }
+      cur_entry++;
+    } // per entry
+    obj_lookup_page++;
+  } // per object lookup page
+  SPIFFS_GC_DBG("gc_check: wipe pallo:%d pdele:%d\n", allo, dele);
+  fs->stats_p_allocated -= allo;
+  fs->stats_p_deleted -= dele;
+  return res;
+}
+
+// Finds block candidates to erase
+s32_t spiffs_gc_find_candidate(
+    spiffs *fs,
+    spiffs_block_ix **block_candidates,
+    int *candidate_count) {
+  s32_t res = SPIFFS_OK;
+  u32_t blocks = fs->block_count;
+  spiffs_block_ix cur_block = 0;
+  u32_t cur_block_addr = 0;
+  spiffs_obj_id *obj_lu_buf = (spiffs_obj_id *)fs->lu_work;
+  int cur_entry = 0;
+
+  // using fs->work area as sorted candidate memory, (spiffs_block_ix)cand_bix/(s32_t)score
+  int max_candidates = MIN(fs->block_count, (SPIFFS_CFG_LOG_PAGE_SZ(fs)-8)/(sizeof(spiffs_block_ix) + sizeof(s32_t)));
+  *candidate_count = 0;
+  c_memset(fs->work, 0xff, SPIFFS_CFG_LOG_PAGE_SZ(fs));
+
+  // divide up work area into block indices and scores
+  // todo alignment?
+  spiffs_block_ix *cand_blocks = (spiffs_block_ix *)fs->work;
+  s32_t *cand_scores = (s32_t *)(fs->work + max_candidates * sizeof(spiffs_block_ix));
+
+  *block_candidates = cand_blocks;
+
+  int entries_per_page = (SPIFFS_CFG_LOG_PAGE_SZ(fs) / sizeof(spiffs_obj_id));
+
+  // check each block
+  while (res == SPIFFS_OK && blocks--) {
+    u16_t deleted_pages_in_block = 0;
+    u16_t used_pages_in_block = 0;
+    int obj_lookup_page = 0;
+    // check each object lookup page
+    while (res == SPIFFS_OK && obj_lookup_page < (int)SPIFFS_OBJ_LOOKUP_PAGES(fs)) {
+      int entry_offset = obj_lookup_page * entries_per_page;
+      res = _spiffs_rd(fs, SPIFFS_OP_T_OBJ_LU | SPIFFS_OP_C_READ,
+          0, cur_block_addr + SPIFFS_PAGE_TO_PADDR(fs, obj_lookup_page), SPIFFS_CFG_LOG_PAGE_SZ(fs), fs->lu_work);
+      // check each entry
+      while (res == SPIFFS_OK &&
+          cur_entry - entry_offset < entries_per_page &&
+          cur_entry < (int)(SPIFFS_PAGES_PER_BLOCK(fs)-SPIFFS_OBJ_LOOKUP_PAGES(fs))) {
+        spiffs_obj_id obj_id = obj_lu_buf[cur_entry-entry_offset];
+        if (obj_id == SPIFFS_OBJ_ID_FREE) {
+          // when a free entry is encountered, scan logic ensures that all following entries are free also
+          res = 1; // kill object lu loop
+          break;
+        } else  if (obj_id == SPIFFS_OBJ_ID_DELETED) {
+          deleted_pages_in_block++;
+        } else {
+          used_pages_in_block++;
+        }
+        cur_entry++;
+      } // per entry
+      obj_lookup_page++;
+    } // per object lookup page
+    if (res == 1) res = SPIFFS_OK;
+
+    // calculate score and insert into candidate table
+    // stoneage sort, but probably not so many blocks
+    if (res == SPIFFS_OK && deleted_pages_in_block > 0) {
+      // read erase count
+      spiffs_obj_id erase_count;
+      res = _spiffs_rd(fs, SPIFFS_OP_C_READ | SPIFFS_OP_T_OBJ_LU2, 0,
+          SPIFFS_ERASE_COUNT_PADDR(fs, cur_block),
+          sizeof(spiffs_obj_id), (u8_t *)&erase_count);
+      SPIFFS_CHECK_RES(res);
+
+      spiffs_obj_id erase_age;
+      if (fs->max_erase_count > erase_count) {
+        erase_age = fs->max_erase_count - erase_count;
+      } else {
+        erase_age = SPIFFS_OBJ_ID_FREE - (erase_count - fs->max_erase_count);
+      }
+
+      s32_t score =
+          deleted_pages_in_block * SPIFFS_GC_HEUR_W_DELET +
+          used_pages_in_block * SPIFFS_GC_HEUR_W_USED +
+          erase_age * SPIFFS_GC_HEUR_W_ERASE_AGE;
+      int cand_ix = 0;
+      SPIFFS_GC_DBG("\ngc_check: bix:%d del:%d use:%d score:%d\n", cur_block, deleted_pages_in_block, used_pages_in_block, score);
+      while (cand_ix < max_candidates) {
+        if (cand_blocks[cand_ix] == (spiffs_block_ix)-1) {
+          cand_blocks[cand_ix] = cur_block;
+          cand_scores[cand_ix] = score;
+          break;
+        } else if (cand_scores[cand_ix] < score) {
+          int reorder_cand_ix = max_candidates - 2;
+          while (reorder_cand_ix >= cand_ix) {
+            cand_blocks[reorder_cand_ix + 1] = cand_blocks[reorder_cand_ix];
+            cand_scores[reorder_cand_ix + 1] = cand_scores[reorder_cand_ix];
+            reorder_cand_ix--;
+          }
+          cand_blocks[cand_ix] = cur_block;
+          cand_scores[cand_ix] = score;
+          break;
+        }
+        cand_ix++;
+      }
+      (*candidate_count)++;
+    }
+
+    cur_entry = 0;
+    cur_block++;
+    cur_block_addr += SPIFFS_CFG_LOG_BLOCK_SZ(fs);
+  } // per block
+
+  return res;
+}
+
+typedef enum {
+  FIND_OBJ_DATA,
+  MOVE_OBJ_DATA,
+  MOVE_OBJ_IX,
+  FINISHED
+} spiffs_gc_clean_state;
+
+typedef struct {
+  spiffs_gc_clean_state state;
+  spiffs_obj_id cur_obj_id;
+  spiffs_span_ix cur_objix_spix;
+  spiffs_page_ix cur_objix_pix;
+  int stored_scan_entry_index;
+  u8_t obj_id_found;
+} spiffs_gc;
+
+// Empties given block by moving all data into free pages of another block
+// Strategy:
+//   loop:
+//   scan object lookup for object data pages
+//   for first found id, check spix and load corresponding object index page to memory
+//   push object scan lookup entry index
+//     rescan object lookup, find data pages with same id and referenced by same object index
+//     move data page, update object index in memory
+//     when reached end of lookup, store updated object index
+//   pop object scan lookup entry index
+//   repeat loop until end of object lookup
+//   scan object lookup again for remaining object index pages, move to new page in other block
+//
+s32_t spiffs_gc_clean(spiffs *fs, spiffs_block_ix bix) {
+  s32_t res = SPIFFS_OK;
+  int entries_per_page = (SPIFFS_CFG_LOG_PAGE_SZ(fs) / sizeof(spiffs_obj_id));
+  int cur_entry = 0;
+  spiffs_obj_id *obj_lu_buf = (spiffs_obj_id *)fs->lu_work;
+  spiffs_gc gc;
+  spiffs_page_ix cur_pix = 0;
+  spiffs_page_object_ix_header *objix_hdr = (spiffs_page_object_ix_header *)fs->work;
+  spiffs_page_object_ix *objix = (spiffs_page_object_ix *)fs->work;
+
+  SPIFFS_GC_DBG("gc_clean: cleaning block %d\n", bix);
+
+  c_memset(&gc, 0, sizeof(spiffs_gc));
+  gc.state = FIND_OBJ_DATA;
+
+  if (fs->free_cursor_block_ix == bix) {
+    // move free cursor to next block, cannot use free pages from the block we want to clean
+    fs->free_cursor_block_ix = (bix+1)%fs->block_count;
+    fs->free_cursor_obj_lu_entry = 0;
+    SPIFFS_GC_DBG("gc_clean: move free cursor to block %d\n", fs->free_cursor_block_ix);
+  }
+
+  while (res == SPIFFS_OK && gc.state != FINISHED) {
+    SPIFFS_GC_DBG("gc_clean: state = %d entry:%d\n", gc.state, cur_entry);
+    gc.obj_id_found = 0;
+
+    // scan through lookup pages
+    int obj_lookup_page = cur_entry / entries_per_page;
+    u8_t scan = 1;
+    // check each object lookup page
+    while (scan && res == SPIFFS_OK && obj_lookup_page < (int)SPIFFS_OBJ_LOOKUP_PAGES(fs)) {
+      int entry_offset = obj_lookup_page * entries_per_page;
+      res = _spiffs_rd(fs, SPIFFS_OP_T_OBJ_LU | SPIFFS_OP_C_READ,
+          0, bix * SPIFFS_CFG_LOG_BLOCK_SZ(fs) + SPIFFS_PAGE_TO_PADDR(fs, obj_lookup_page),
+          SPIFFS_CFG_LOG_PAGE_SZ(fs), fs->lu_work);
+      // check each entry
+      while (scan && res == SPIFFS_OK &&
+          cur_entry - entry_offset < entries_per_page && cur_entry < (int)(SPIFFS_PAGES_PER_BLOCK(fs)-SPIFFS_OBJ_LOOKUP_PAGES(fs))) {
+        spiffs_obj_id obj_id = obj_lu_buf[cur_entry-entry_offset];
+        cur_pix = SPIFFS_OBJ_LOOKUP_ENTRY_TO_PIX(fs, bix, cur_entry);
+
+        // act upon object id depending on gc state
+        switch (gc.state) {
+        case FIND_OBJ_DATA:
+          if (obj_id != SPIFFS_OBJ_ID_DELETED && obj_id != SPIFFS_OBJ_ID_FREE &&
+              ((obj_id & SPIFFS_OBJ_ID_IX_FLAG) == 0)) {
+            SPIFFS_GC_DBG("gc_clean: FIND_DATA state:%d - found obj id %04x\n", gc.state, obj_id);
+            gc.obj_id_found = 1;
+            gc.cur_obj_id = obj_id;
+            scan = 0;
+          }
+          break;
+        case MOVE_OBJ_DATA:
+          if (obj_id == gc.cur_obj_id) {
+            spiffs_page_header p_hdr;
+            res = _spiffs_rd(fs, SPIFFS_OP_T_OBJ_LU2 | SPIFFS_OP_C_READ,
+                0, SPIFFS_PAGE_TO_PADDR(fs, cur_pix), sizeof(spiffs_page_header), (u8_t*)&p_hdr);
+            SPIFFS_CHECK_RES(res);
+            SPIFFS_GC_DBG("gc_clean: MOVE_DATA found data page %04x:%04x @ %04x\n", gc.cur_obj_id, p_hdr.span_ix, cur_pix);
+            if (SPIFFS_OBJ_IX_ENTRY_SPAN_IX(fs, p_hdr.span_ix) != gc.cur_objix_spix) {
+              SPIFFS_GC_DBG("gc_clean: MOVE_DATA no objix spix match, take in another run\n");
+            } else {
+              spiffs_page_ix new_data_pix;
+              if (p_hdr.flags & SPIFFS_PH_FLAG_DELET) {
+                // move page
+                res = spiffs_page_move(fs, 0, 0, obj_id, &p_hdr, cur_pix, &new_data_pix);
+                SPIFFS_GC_DBG("gc_clean: MOVE_DATA move objix %04x:%04x page %04x to %04x\n", gc.cur_obj_id, p_hdr.span_ix, cur_pix, new_data_pix);
+                SPIFFS_CHECK_RES(res);
+                // move wipes obj_lu, reload it
+                res = _spiffs_rd(fs, SPIFFS_OP_T_OBJ_LU | SPIFFS_OP_C_READ,
+                    0, bix * SPIFFS_CFG_LOG_BLOCK_SZ(fs) + SPIFFS_PAGE_TO_PADDR(fs, obj_lookup_page),
+                    SPIFFS_CFG_LOG_PAGE_SZ(fs), fs->lu_work);
+                SPIFFS_CHECK_RES(res);
+              } else {
+                // page is deleted but not deleted in lookup, scrap it
+                SPIFFS_GC_DBG("gc_clean: MOVE_DATA wipe objix %04x:%04x page %04x\n", obj_id, p_hdr.span_ix, cur_pix);
+                res = spiffs_page_delete(fs, cur_pix);
+                SPIFFS_CHECK_RES(res);
+                new_data_pix = SPIFFS_OBJ_ID_FREE;
+              }
+              // update memory representation of object index page with new data page
+              if (gc.cur_objix_spix == 0) {
+                // update object index header page
+                ((spiffs_page_ix*)((u8_t *)objix_hdr + sizeof(spiffs_page_object_ix_header)))[p_hdr.span_ix] = new_data_pix;
+                SPIFFS_GC_DBG("gc_clean: MOVE_DATA wrote page %04x to objix_hdr entry %02x in mem\n", new_data_pix, SPIFFS_OBJ_IX_ENTRY(fs, p_hdr.span_ix));
+              } else {
+                // update object index page
+                ((spiffs_page_ix*)((u8_t *)objix + sizeof(spiffs_page_object_ix)))[SPIFFS_OBJ_IX_ENTRY(fs, p_hdr.span_ix)] = new_data_pix;
+                SPIFFS_GC_DBG("gc_clean: MOVE_DATA wrote page %04x to objix entry %02x in mem\n", new_data_pix, SPIFFS_OBJ_IX_ENTRY(fs, p_hdr.span_ix));
+              }
+            }
+          }
+          break;
+        case MOVE_OBJ_IX:
+          if (obj_id != SPIFFS_OBJ_ID_DELETED && obj_id != SPIFFS_OBJ_ID_FREE &&
+              (obj_id & SPIFFS_OBJ_ID_IX_FLAG)) {
+            // found an index object id
+            spiffs_page_header p_hdr;
+            spiffs_page_ix new_pix;
+            // load header
+            res = _spiffs_rd(fs, SPIFFS_OP_T_OBJ_LU2 | SPIFFS_OP_C_READ,
+                0, SPIFFS_PAGE_TO_PADDR(fs, cur_pix), sizeof(spiffs_page_header), (u8_t*)&p_hdr);
+            SPIFFS_CHECK_RES(res);
+            if (p_hdr.flags & SPIFFS_PH_FLAG_DELET) {
+              // move page
+              res = spiffs_page_move(fs, 0, 0, obj_id, &p_hdr, cur_pix, &new_pix);
+              SPIFFS_GC_DBG("gc_clean: MOVE_OBJIX move objix %04x:%04x page %04x to %04x\n", obj_id, p_hdr.span_ix, cur_pix, new_pix);
+              SPIFFS_CHECK_RES(res);
+              spiffs_cb_object_event(fs, 0, SPIFFS_EV_IX_UPD, obj_id, p_hdr.span_ix, new_pix, 0);
+              // move wipes obj_lu, reload it
+              res = _spiffs_rd(fs, SPIFFS_OP_T_OBJ_LU | SPIFFS_OP_C_READ,
+                  0, bix * SPIFFS_CFG_LOG_BLOCK_SZ(fs) + SPIFFS_PAGE_TO_PADDR(fs, obj_lookup_page),
+                  SPIFFS_CFG_LOG_PAGE_SZ(fs), fs->lu_work);
+              SPIFFS_CHECK_RES(res);
+            } else {
+              // page is deleted but not deleted in lookup, scrap it
+              SPIFFS_GC_DBG("gc_clean: MOVE_OBJIX wipe objix %04x:%04x page %04x\n", obj_id, p_hdr.span_ix, cur_pix);
+              res = spiffs_page_delete(fs, cur_pix);
+              if (res == SPIFFS_OK) {
+                spiffs_cb_object_event(fs, 0, SPIFFS_EV_IX_DEL, obj_id, p_hdr.span_ix, cur_pix, 0);
+              }
+            }
+            SPIFFS_CHECK_RES(res);
+          }
+          break;
+        default:
+          scan = 0;
+          break;
+        }
+        cur_entry++;
+      } // per entry
+      obj_lookup_page++;
+    } // per object lookup page
+
+    if (res != SPIFFS_OK) break;
+
+    // state finalization and switch
+    switch (gc.state) {
+    case FIND_OBJ_DATA:
+      if (gc.obj_id_found) {
+        // find out corresponding obj ix page and load it to memory
+        spiffs_page_header p_hdr;
+        spiffs_page_ix objix_pix;
+        gc.stored_scan_entry_index = cur_entry;
+        cur_entry = 0;
+        gc.state = MOVE_OBJ_DATA;
+        res = _spiffs_rd(fs, SPIFFS_OP_T_OBJ_LU2 | SPIFFS_OP_C_READ,
+            0, SPIFFS_PAGE_TO_PADDR(fs, cur_pix), sizeof(spiffs_page_header), (u8_t*)&p_hdr);
+        SPIFFS_CHECK_RES(res);
+        gc.cur_objix_spix = SPIFFS_OBJ_IX_ENTRY_SPAN_IX(fs, p_hdr.span_ix);
+        SPIFFS_GC_DBG("gc_clean: FIND_DATA find objix span_ix:%04x\n", gc.cur_objix_spix);
+        res = spiffs_obj_lu_find_id_and_span(fs, gc.cur_obj_id | SPIFFS_OBJ_ID_IX_FLAG, gc.cur_objix_spix, 0, &objix_pix);
+        SPIFFS_CHECK_RES(res);
+        SPIFFS_GC_DBG("gc_clean: FIND_DATA found object index at page %04x\n", objix_pix);
+        res = _spiffs_rd(fs, SPIFFS_OP_T_OBJ_LU2 | SPIFFS_OP_C_READ,
+            0, SPIFFS_PAGE_TO_PADDR(fs, objix_pix), SPIFFS_CFG_LOG_PAGE_SZ(fs), fs->work);
+        SPIFFS_CHECK_RES(res);
+        SPIFFS_VALIDATE_OBJIX(objix->p_hdr, gc.cur_obj_id | SPIFFS_OBJ_ID_IX_FLAG, gc.cur_objix_spix);
+        gc.cur_objix_pix = objix_pix;
+      } else {
+        gc.state = MOVE_OBJ_IX;
+        cur_entry = 0; // restart entry scan index
+      }
+      break;
+    case MOVE_OBJ_DATA: {
+      // store modified objix (hdr) page
+      spiffs_page_ix new_objix_pix;
+      gc.state = FIND_OBJ_DATA;
+      cur_entry = gc.stored_scan_entry_index;
+      if (gc.cur_objix_spix == 0) {
+        // store object index header page
+        res = spiffs_object_update_index_hdr(fs, 0, gc.cur_obj_id | SPIFFS_OBJ_ID_IX_FLAG, gc.cur_objix_pix, fs->work, 0, 0, &new_objix_pix);
+        SPIFFS_GC_DBG("gc_clean: MOVE_DATA store modified objix_hdr page, %04x:%04x\n", new_objix_pix, 0);
+        SPIFFS_CHECK_RES(res);
+      } else {
+        // store object index page
+        res = spiffs_page_move(fs, 0, fs->work, gc.cur_obj_id | SPIFFS_OBJ_ID_IX_FLAG, 0, gc.cur_objix_pix, &new_objix_pix);
+        SPIFFS_GC_DBG("gc_clean: MOVE_DATA store modified objix page, %04x:%04x\n", new_objix_pix, objix->p_hdr.span_ix);
+        SPIFFS_CHECK_RES(res);
+        spiffs_cb_object_event(fs, 0, SPIFFS_EV_IX_UPD, gc.cur_obj_id, objix->p_hdr.span_ix, new_objix_pix, 0);
+      }
+    }
+    break;
+    case MOVE_OBJ_IX:
+      gc.state = FINISHED;
+      break;
+    default:
+      cur_entry = 0;
+      break;
+    }
+    SPIFFS_GC_DBG("gc_clean: state-> %d\n", gc.state);
+  } // while state != FINISHED
+
+
+  return res;
+}
+

cores/esp8266/spiffs/spiffs_hydrogen.c

@@ -0,0 +1,871 @@
+/*
+ * spiffs_hydrogen.c
+ *
+ *  Created on: Jun 16, 2013
+ *      Author: petera
+ */
+
+#include "spiffs.h"
+#include "spiffs_nucleus.h"
+
+static s32_t spiffs_fflush_cache(spiffs *fs, spiffs_file fh);
+
+#if SPIFFS_BUFFER_HELP
+u32_t SPIFFS_buffer_bytes_for_filedescs(spiffs *fs, u32_t num_descs) {
+  return num_descs * sizeof(spiffs_fd);
+}
+#if SPIFFS_CACHE
+u32_t SPIFFS_buffer_bytes_for_cache(spiffs *fs, u32_t num_pages) {
+  return sizeof(spiffs_cache) + num_pages * (sizeof(spiffs_cache_page) + SPIFFS_CFG_LOG_PAGE_SZ(fs));
+}
+#endif
+#endif
+
+s32_t SPIFFS_mount(spiffs *fs, spiffs_config *config, u8_t *work,
+    u8_t *fd_space, u32_t fd_space_size,
+    void *cache, u32_t cache_size,
+    spiffs_check_callback check_cb_f) {
+  SPIFFS_LOCK(fs);
+  c_memset(fs, 0, sizeof(spiffs));
+  c_memcpy(&fs->cfg, config, sizeof(spiffs_config));
+  fs->block_count = SPIFFS_CFG_PHYS_SZ(fs) / SPIFFS_CFG_LOG_BLOCK_SZ(fs);
+  fs->work = &work[0];
+  fs->lu_work = &work[SPIFFS_CFG_LOG_PAGE_SZ(fs)];
+  c_memset(fd_space, 0, fd_space_size);
+  // align fd_space pointer to pointer size byte boundary, below is safe
+  u8_t ptr_size = sizeof(void*);
+// #pragma GCC diagnostic push
+// #pragma GCC diagnostic ignored "-Wpointer-to-int-cast"
+  u8_t addr_lsb = (u8_t)(((u32_t)fd_space) & (ptr_size-1));
+// #pragma GCC diagnostic pop
+  if (addr_lsb) {
+    fd_space += (ptr_size-addr_lsb);
+    fd_space_size -= (ptr_size-addr_lsb);
+  }
+  fs->fd_space = fd_space;
+  fs->fd_count = (fd_space_size/sizeof(spiffs_fd));
+
+  // align cache pointer to 4 byte boundary, below is safe
+// #pragma GCC diagnostic push
+// #pragma GCC diagnostic ignored "-Wpointer-to-int-cast"
+  addr_lsb = (u8_t)(((u32_t)cache) & (ptr_size-1));
+// #pragma GCC diagnostic pop
+  if (addr_lsb) {
+    u8_t *cache_8 = (u8_t *)cache;
+    cache_8 += (ptr_size-addr_lsb);
+    cache = cache_8;
+    cache_size -= (ptr_size-addr_lsb);
+  }
+  if (cache_size & (ptr_size-1)) {
+    cache_size -= (cache_size & (ptr_size-1));
+  }
+#if SPIFFS_CACHE
+  fs->cache = cache;
+  fs->cache_size = cache_size;
+  spiffs_cache_init(fs);
+#endif
+
+  s32_t res = spiffs_obj_lu_scan(fs);
+  SPIFFS_API_CHECK_RES_UNLOCK(fs, res);
+
+  SPIFFS_DBG("page index byte len:         %d\n", SPIFFS_CFG_LOG_PAGE_SZ(fs));
+  SPIFFS_DBG("object lookup pages:         %d\n", SPIFFS_OBJ_LOOKUP_PAGES(fs));
+  SPIFFS_DBG("page pages per block:        %d\n", SPIFFS_PAGES_PER_BLOCK(fs));
+  SPIFFS_DBG("page header length:          %d\n", sizeof(spiffs_page_header));
+  SPIFFS_DBG("object header index entries: %d\n", SPIFFS_OBJ_HDR_IX_LEN(fs));
+  SPIFFS_DBG("object index entries:        %d\n", SPIFFS_OBJ_IX_LEN(fs));
+  SPIFFS_DBG("available file descriptors:  %d\n", fs->fd_count);
+  SPIFFS_DBG("free blocks:                 %d\n", fs->free_blocks);
+
+  fs->check_cb_f = check_cb_f;
+
+  SPIFFS_UNLOCK(fs);
+
+  return 0;
+}
+
+void SPIFFS_unmount(spiffs *fs) {
+  if (!SPIFFS_CHECK_MOUNT(fs)) return;
+  SPIFFS_LOCK(fs);
+  u32_t i;
+  spiffs_fd *fds = (spiffs_fd *)fs->fd_space;
+  for (i = 0; i < fs->fd_count; i++) {
+    spiffs_fd *cur_fd = &fds[i];
+    if (cur_fd->file_nbr != 0) {
+#if SPIFFS_CACHE
+      (void)spiffs_fflush_cache(fs, cur_fd->file_nbr);
+#endif
+      spiffs_fd_return(fs, cur_fd->file_nbr);
+    }
+  }
+  fs->block_count = 0;
+  SPIFFS_UNLOCK(fs);
+}
+
+s32_t SPIFFS_errno(spiffs *fs) {
+  return fs->errno;
+}
+
+s32_t SPIFFS_creat(spiffs *fs, const char *path, spiffs_mode mode) {
+  (void)mode;
+  SPIFFS_API_CHECK_MOUNT(fs);
+  SPIFFS_LOCK(fs);
+  spiffs_obj_id obj_id;
+  s32_t res;
+
+  res = spiffs_obj_lu_find_free_obj_id(fs, &obj_id, (u8_t *)path);
+  SPIFFS_API_CHECK_RES_UNLOCK(fs, res);
+  res = spiffs_object_create(fs, obj_id, (u8_t *)path, SPIFFS_TYPE_FILE, 0);
+  SPIFFS_API_CHECK_RES_UNLOCK(fs, res);
+  SPIFFS_UNLOCK(fs);
+  return 0;
+}
+
+spiffs_file SPIFFS_open(spiffs *fs, const char *path, spiffs_flags flags, spiffs_mode mode) {
+  (void)mode;
+  SPIFFS_API_CHECK_MOUNT(fs);
+  SPIFFS_LOCK(fs);
+
+  spiffs_fd *fd;
+  spiffs_page_ix pix;
+
+  s32_t res = spiffs_fd_find_new(fs, &fd);
+  SPIFFS_API_CHECK_RES_UNLOCK(fs, res);
+
+  res = spiffs_object_find_object_index_header_by_name(fs, (u8_t*)path, &pix);
+  if ((flags & SPIFFS_CREAT) == 0) {
+    if (res < SPIFFS_OK) {
+      spiffs_fd_return(fs, fd->file_nbr);
+    }
+    SPIFFS_API_CHECK_RES_UNLOCK(fs, res);
+  }
+
+  if ((flags & SPIFFS_CREAT) && res == SPIFFS_ERR_NOT_FOUND) {
+    spiffs_obj_id obj_id;
+    // no need to enter conflicting name here, already looked for it above
+    res = spiffs_obj_lu_find_free_obj_id(fs, &obj_id, 0);
+    if (res < SPIFFS_OK) {
+      spiffs_fd_return(fs, fd->file_nbr);
+    }
+    SPIFFS_API_CHECK_RES_UNLOCK(fs, res);
+    res = spiffs_object_create(fs, obj_id, (u8_t*)path, SPIFFS_TYPE_FILE, &pix);
+    if (res < SPIFFS_OK) {
+      spiffs_fd_return(fs, fd->file_nbr);
+    }
+    SPIFFS_API_CHECK_RES_UNLOCK(fs, res);
+    flags &= ~SPIFFS_TRUNC;
+  } else {
+    if (res < SPIFFS_OK) {
+      spiffs_fd_return(fs, fd->file_nbr);
+    }
+    SPIFFS_API_CHECK_RES_UNLOCK(fs, res);
+  }
+  res = spiffs_object_open_by_page(fs, pix, fd, flags, mode);
+  if (res < SPIFFS_OK) {
+    spiffs_fd_return(fs, fd->file_nbr);
+  }
+  SPIFFS_API_CHECK_RES_UNLOCK(fs, res);
+  if (flags & SPIFFS_TRUNC) {
+    res = spiffs_object_truncate(fd, 0, 0);
+    if (res < SPIFFS_OK) {
+      spiffs_fd_return(fs, fd->file_nbr);
+    }
+    SPIFFS_API_CHECK_RES_UNLOCK(fs, res);
+  }
+
+  fd->fdoffset = 0;
+
+  SPIFFS_UNLOCK(fs);
+
+  return fd->file_nbr;
+}
+
+spiffs_file SPIFFS_open_by_dirent(spiffs *fs, struct spiffs_dirent *e, spiffs_flags flags, spiffs_mode mode) {
+  SPIFFS_API_CHECK_MOUNT(fs);
+  SPIFFS_LOCK(fs);
+
+  spiffs_fd *fd;
+
+  s32_t res = spiffs_fd_find_new(fs, &fd);
+  SPIFFS_API_CHECK_RES_UNLOCK(fs, res);
+
+  res = spiffs_object_open_by_page(fs, e->pix, fd, flags, mode);
+  if (res < SPIFFS_OK) {
+    spiffs_fd_return(fs, fd->file_nbr);
+  }
+  SPIFFS_API_CHECK_RES_UNLOCK(fs, res);
+  if (flags & SPIFFS_TRUNC) {
+    res = spiffs_object_truncate(fd, 0, 0);
+    if (res < SPIFFS_OK) {
+      spiffs_fd_return(fs, fd->file_nbr);
+    }
+    SPIFFS_API_CHECK_RES_UNLOCK(fs, res);
+  }
+
+  fd->fdoffset = 0;
+
+  SPIFFS_UNLOCK(fs);
+
+  return fd->file_nbr;
+}
+
+s32_t SPIFFS_read(spiffs *fs, spiffs_file fh, void *buf, u32_t len) {
+  SPIFFS_API_CHECK_MOUNT(fs);
+  SPIFFS_LOCK(fs);
+
+  spiffs_fd *fd;
+  s32_t res;
+
+  res = spiffs_fd_get(fs, fh, &fd);
+  SPIFFS_API_CHECK_RES_UNLOCK(fs, res);
+
+  if ((fd->flags & SPIFFS_RDONLY) == 0) {
+    res = SPIFFS_ERR_NOT_READABLE;
+    SPIFFS_API_CHECK_RES_UNLOCK(fs, res);
+  }
+
+#if SPIFFS_CACHE_WR
+  spiffs_fflush_cache(fs, fh);
+#endif
+
+  if (fd->fdoffset + len >= fd->size) {
+    // reading beyond file size
+    s32_t avail = fd->size - fd->fdoffset;
+    if (avail <= 0) {
+      SPIFFS_API_CHECK_RES_UNLOCK(fs, SPIFFS_ERR_END_OF_OBJECT);
+    }
+    res = spiffs_object_read(fd, fd->fdoffset, avail, (u8_t*)buf);
+    if (res == SPIFFS_ERR_END_OF_OBJECT) {
+      fd->fdoffset += avail;
+      SPIFFS_UNLOCK(fs);
+      return avail;
+    } else {
+      SPIFFS_API_CHECK_RES_UNLOCK(fs, res);
+      len = avail;
+    }
+  } else {
+    // reading within file size
+    res = spiffs_object_read(fd, fd->fdoffset, len, (u8_t*)buf);
+    SPIFFS_API_CHECK_RES_UNLOCK(fs, res);
+  }
+  fd->fdoffset += len;
+
+  SPIFFS_UNLOCK(fs);
+
+  return len;
+}
+
+static s32_t spiffs_hydro_write(spiffs *fs, spiffs_fd *fd, void *buf, u32_t offset, s32_t len) {
+  (void)fs;
+  s32_t res = SPIFFS_OK;
+  s32_t remaining = len;
+  if (fd->size != SPIFFS_UNDEFINED_LEN && offset < fd->size) {
+    s32_t m_len = MIN((s32_t)(fd->size - offset), len);
+    res = spiffs_object_modify(fd, offset, (u8_t *)buf, m_len);
+    SPIFFS_CHECK_RES(res);
+    remaining -= m_len;
+    u8_t *buf_8 = (u8_t *)buf;
+    buf_8 += m_len;
+    buf = buf_8;
+    offset += m_len;
+  }
+  if (remaining > 0) {
+    res = spiffs_object_append(fd, offset, (u8_t *)buf, remaining);
+    SPIFFS_CHECK_RES(res);
+  }
+  return len;
+
+}
+
+s32_t SPIFFS_write(spiffs *fs, spiffs_file fh, void *buf, u32_t len) {
+  SPIFFS_API_CHECK_MOUNT(fs);
+  SPIFFS_LOCK(fs);
+
+  spiffs_fd *fd;
+  s32_t res;
+  u32_t offset;
+
+  res = spiffs_fd_get(fs, fh, &fd);
+  SPIFFS_API_CHECK_RES_UNLOCK(fs, res);
+
+  if ((fd->flags & SPIFFS_WRONLY) == 0) {
+    res = SPIFFS_ERR_NOT_WRITABLE;
+    SPIFFS_API_CHECK_RES_UNLOCK(fs, res);
+  }
+
+  offset = fd->fdoffset;
+
+#if SPIFFS_CACHE_WR
+  if (fd->cache_page == 0) {
+    // see if object id is associated with cache already
+    fd->cache_page = spiffs_cache_page_get_by_fd(fs, fd);
+  }
+#endif
+  if (fd->flags & SPIFFS_APPEND) {
+    if (fd->size == SPIFFS_UNDEFINED_LEN) {
+      offset = 0;
+    } else {
+      offset = fd->size;
+    }
+#if SPIFFS_CACHE_WR
+    if (fd->cache_page) {
+      offset = MAX(offset, fd->cache_page->offset + fd->cache_page->size);
+    }
+#endif
+  }
+
+  SPIFFS_DBG("SPIFFS_write %d %04x offs:%d len %d\n", fh, fd->obj_id, offset, len);
+
+#if SPIFFS_CACHE_WR
+  if ((fd->flags & SPIFFS_DIRECT) == 0) {
+    if (len < (s32_t)SPIFFS_CFG_LOG_PAGE_SZ(fs)) {
+      // small write, try to cache it
+      u8_t alloc_cpage = 1;
+      if (fd->cache_page) {
+        // have a cached page for this fd already, check cache page boundaries
+        if (offset < fd->cache_page->offset || // writing before cache
+            offset > fd->cache_page->offset + fd->cache_page->size || // writing after cache
+            offset + len > fd->cache_page->offset + SPIFFS_CFG_LOG_PAGE_SZ(fs)) // writing beyond cache page
+        {
+          // boundary violation, write back cache first and allocate new
+          SPIFFS_CACHE_DBG("CACHE_WR_DUMP: dumping cache page %d for fd %d:&04x, boundary viol, offs:%d size:%d\n",
+              fd->cache_page->ix, fd->file_nbr, fd->obj_id, fd->cache_page->offset, fd->cache_page->size);
+          res = spiffs_hydro_write(fs, fd,
+              spiffs_get_cache_page(fs, spiffs_get_cache(fs), fd->cache_page->ix),
+              fd->cache_page->offset, fd->cache_page->size);
+          spiffs_cache_fd_release(fs, fd->cache_page);
+        } else {
+          // writing within cache
+          alloc_cpage = 0;
+        }
+      }
+
+      if (alloc_cpage) {
+        fd->cache_page = spiffs_cache_page_allocate_by_fd(fs, fd);
+        if (fd->cache_page) {
+          fd->cache_page->offset = offset;
+          fd->cache_page->size = 0;
+          SPIFFS_CACHE_DBG("CACHE_WR_ALLO: allocating cache page %d for fd %d:%04x\n",
+              fd->cache_page->ix, fd->file_nbr, fd->obj_id);
+        }
+      }
+
+      if (fd->cache_page) {
+        u32_t offset_in_cpage = offset - fd->cache_page->offset;
+        SPIFFS_CACHE_DBG("CACHE_WR_WRITE: storing to cache page %d for fd %d:%04x, offs %d:%d len %d\n",
+            fd->cache_page->ix, fd->file_nbr, fd->obj_id,
+            offset, offset_in_cpage, len);
+        spiffs_cache *cache = spiffs_get_cache(fs);
+        u8_t *cpage_data = spiffs_get_cache_page(fs, cache, fd->cache_page->ix);
+        c_memcpy(&cpage_data[offset_in_cpage], buf, len);
+        fd->cache_page->size = MAX(fd->cache_page->size, offset_in_cpage + len);
+        fd->fdoffset += len;
+        SPIFFS_UNLOCK(fs);
+        return len;
+      } else {
+        res = spiffs_hydro_write(fs, fd, buf, offset, len);
+        SPIFFS_API_CHECK_RES(fs, res);
+        fd->fdoffset += len;
+        SPIFFS_UNLOCK(fs);
+        return res;
+      }
+    } else {
+      // big write, no need to cache it - but first check if there is a cached write already
+      if (fd->cache_page) {
+        // write back cache first
+        SPIFFS_CACHE_DBG("CACHE_WR_DUMP: dumping cache page %d for fd %d:%04x, big write, offs:%d size:%d\n",
+            fd->cache_page->ix, fd->file_nbr, fd->obj_id, fd->cache_page->offset, fd->cache_page->size);
+        res = spiffs_hydro_write(fs, fd,
+            spiffs_get_cache_page(fs, spiffs_get_cache(fs), fd->cache_page->ix),
+            fd->cache_page->offset, fd->cache_page->size);
+        spiffs_cache_fd_release(fs, fd->cache_page);
+        res = spiffs_hydro_write(fs, fd, buf, offset, len);
+        SPIFFS_API_CHECK_RES(fs, res);
+      }
+    }
+  }
+#endif
+
+  res = spiffs_hydro_write(fs, fd, buf, offset, len);
+  SPIFFS_API_CHECK_RES(fs, res);
+  fd->fdoffset += len;
+
+  SPIFFS_UNLOCK(fs);
+
+  return res;
+}
+
+s32_t SPIFFS_lseek(spiffs *fs, spiffs_file fh, s32_t offs, int whence) {
+  SPIFFS_API_CHECK_MOUNT(fs);
+  SPIFFS_LOCK(fs);
+
+  spiffs_fd *fd;
+  s32_t res;
+  res = spiffs_fd_get(fs, fh, &fd);
+  SPIFFS_API_CHECK_RES(fs, res);
+
+#if SPIFFS_CACHE_WR
+  spiffs_fflush_cache(fs, fh);
+#endif
+
+  switch (whence) {
+  case SPIFFS_SEEK_CUR:
+    offs = fd->fdoffset+offs;
+    break;
+  case SPIFFS_SEEK_END:
+    offs = (fd->size == SPIFFS_UNDEFINED_LEN ? 0 : fd->size) + offs;
+    break;
+  }
+
+  if (offs > (s32_t)fd->size) {
+    res = SPIFFS_ERR_END_OF_OBJECT;
+  }
+  SPIFFS_API_CHECK_RES_UNLOCK(fs, res);
+
+  spiffs_span_ix data_spix = offs / SPIFFS_DATA_PAGE_SIZE(fs);
+  spiffs_span_ix objix_spix = SPIFFS_OBJ_IX_ENTRY_SPAN_IX(fs, data_spix);
+  if (fd->cursor_objix_spix != objix_spix) {
+    spiffs_page_ix pix;
+    res = spiffs_obj_lu_find_id_and_span(
+        fs, fd->obj_id | SPIFFS_OBJ_ID_IX_FLAG, objix_spix, 0, &pix);
+    SPIFFS_API_CHECK_RES_UNLOCK(fs, res);
+    fd->cursor_objix_spix = objix_spix;
+    fd->cursor_objix_pix = pix;
+  }
+  fd->fdoffset = offs;
+
+  SPIFFS_UNLOCK(fs);
+
+  return 0;
+}
+
+s32_t SPIFFS_remove(spiffs *fs, const char *path) {
+  SPIFFS_API_CHECK_MOUNT(fs);
+  SPIFFS_LOCK(fs);
+
+  spiffs_fd *fd;
+  spiffs_page_ix pix;
+  s32_t res;
+
+  res = spiffs_fd_find_new(fs, &fd);
+  SPIFFS_API_CHECK_RES_UNLOCK(fs, res);
+
+  res = spiffs_object_find_object_index_header_by_name(fs, (u8_t *)path, &pix);
+  if (res != SPIFFS_OK) {
+    spiffs_fd_return(fs, fd->file_nbr);
+  }
+  SPIFFS_API_CHECK_RES_UNLOCK(fs, res);
+
+  res = spiffs_object_open_by_page(fs, pix, fd, 0,0);
+  if (res != SPIFFS_OK) {
+    spiffs_fd_return(fs, fd->file_nbr);
+  }
+  SPIFFS_API_CHECK_RES_UNLOCK(fs, res);
+
+  res = spiffs_object_truncate(fd, 0, 1);
+  if (res != SPIFFS_OK) {
+    spiffs_fd_return(fs, fd->file_nbr);
+  }
+  SPIFFS_API_CHECK_RES_UNLOCK(fs, res);
+
+  SPIFFS_UNLOCK(fs);
+  return 0;
+}
+
+s32_t SPIFFS_fremove(spiffs *fs, spiffs_file fh) {
+  SPIFFS_API_CHECK_MOUNT(fs);
+  SPIFFS_LOCK(fs);
+
+  spiffs_fd *fd;
+  s32_t res;
+  res = spiffs_fd_get(fs, fh, &fd);
+  SPIFFS_API_CHECK_RES_UNLOCK(fs, res);
+
+  if ((fd->flags & SPIFFS_WRONLY) == 0) {
+    res = SPIFFS_ERR_NOT_WRITABLE;
+    SPIFFS_API_CHECK_RES_UNLOCK(fs, res);
+  }
+
+#if SPIFFS_CACHE_WR
+  spiffs_cache_fd_release(fs, fd->cache_page);
+#endif
+
+  res = spiffs_object_truncate(fd, 0, 1);
+
+  SPIFFS_API_CHECK_RES_UNLOCK(fs, res);
+
+  SPIFFS_UNLOCK(fs);
+
+  return 0;
+}
+
+static s32_t spiffs_stat_pix(spiffs *fs, spiffs_page_ix pix, spiffs_file fh, spiffs_stat *s) {
+  spiffs_page_object_ix_header objix_hdr;
+  spiffs_obj_id obj_id;
+  s32_t res =_spiffs_rd(fs,  SPIFFS_OP_T_OBJ_IX | SPIFFS_OP_C_READ, fh,
+      SPIFFS_PAGE_TO_PADDR(fs, pix), sizeof(spiffs_page_object_ix_header), (u8_t *)&objix_hdr);
+  SPIFFS_API_CHECK_RES(fs, res);
+
+  u32_t obj_id_addr = SPIFFS_BLOCK_TO_PADDR(fs, SPIFFS_BLOCK_FOR_PAGE(fs , pix)) +
+      SPIFFS_OBJ_LOOKUP_ENTRY_FOR_PAGE(fs, pix) * sizeof(spiffs_obj_id);
+  res =_spiffs_rd(fs,  SPIFFS_OP_T_OBJ_LU | SPIFFS_OP_C_READ, fh,
+      obj_id_addr, sizeof(spiffs_obj_id), (u8_t *)&obj_id);
+  SPIFFS_API_CHECK_RES(fs, res);
+
+  s->obj_id = obj_id;
+  s->type = objix_hdr.type;
+  s->size = objix_hdr.size == SPIFFS_UNDEFINED_LEN ? 0 : objix_hdr.size;
+  strncpy((char *)s->name, (char *)objix_hdr.name, SPIFFS_OBJ_NAME_LEN);
+
+  return res;
+}
+
+s32_t SPIFFS_stat(spiffs *fs, const char *path, spiffs_stat *s) {
+  SPIFFS_API_CHECK_MOUNT(fs);
+  SPIFFS_LOCK(fs);
+
+  s32_t res;
+  spiffs_page_ix pix;
+
+  res = spiffs_object_find_object_index_header_by_name(fs, (u8_t*)path, &pix);
+  SPIFFS_API_CHECK_RES_UNLOCK(fs, res);
+
+  res = spiffs_stat_pix(fs, pix, 0, s);
+
+  SPIFFS_UNLOCK(fs);
+
+  return res;
+}
+
+s32_t SPIFFS_fstat(spiffs *fs, spiffs_file fh, spiffs_stat *s) {
+  SPIFFS_API_CHECK_MOUNT(fs);
+  SPIFFS_LOCK(fs);
+
+  spiffs_fd *fd;
+  s32_t res;
+
+  res = spiffs_fd_get(fs, fh, &fd);
+  SPIFFS_API_CHECK_RES_UNLOCK(fs, res);
+
+#if SPIFFS_CACHE_WR
+  spiffs_fflush_cache(fs, fh);
+#endif
+
+  res = spiffs_stat_pix(fs, fd->objix_hdr_pix, fh, s);
+
+  SPIFFS_UNLOCK(fs);
+
+  return res;
+}
+
+// Checks if there are any cached writes for the object id associated with
+// given filehandle. If so, these writes are flushed.
+static s32_t spiffs_fflush_cache(spiffs *fs, spiffs_file fh) {
+  s32_t res = SPIFFS_OK;
+#if SPIFFS_CACHE_WR
+
+  spiffs_fd *fd;
+  res = spiffs_fd_get(fs, fh, &fd);
+  SPIFFS_API_CHECK_RES(fs, res);
+
+  if ((fd->flags & SPIFFS_DIRECT) == 0) {
+    if (fd->cache_page == 0) {
+      // see if object id is associated with cache already
+      fd->cache_page = spiffs_cache_page_get_by_fd(fs, fd);
+    }
+    if (fd->cache_page) {
+      SPIFFS_CACHE_DBG("CACHE_WR_DUMP: dumping cache page %d for fd %d:%04x, flush, offs:%d size:%d\n",
+          fd->cache_page->ix, fd->file_nbr,  fd->obj_id, fd->cache_page->offset, fd->cache_page->size);
+      res = spiffs_hydro_write(fs, fd,
+          spiffs_get_cache_page(fs, spiffs_get_cache(fs), fd->cache_page->ix),
+          fd->cache_page->offset, fd->cache_page->size);
+      if (res < SPIFFS_OK) {
+        fs->errno = res;
+      }
+      spiffs_cache_fd_release(fs, fd->cache_page);
+    }
+  }
+#endif
+
+  return res;
+}
+
+s32_t SPIFFS_fflush(spiffs *fs, spiffs_file fh) {
+  SPIFFS_API_CHECK_MOUNT(fs);
+  s32_t res = SPIFFS_OK;
+#if SPIFFS_CACHE_WR
+  SPIFFS_LOCK(fs);
+  res = spiffs_fflush_cache(fs, fh);
+  SPIFFS_API_CHECK_RES_UNLOCK(fs,res);
+  SPIFFS_UNLOCK(fs);
+#endif
+
+  return res;
+}
+
+void SPIFFS_close(spiffs *fs, spiffs_file fh) {
+  if (!SPIFFS_CHECK_MOUNT(fs)) {
+    fs->errno = SPIFFS_ERR_NOT_MOUNTED;
+    return;
+  }
+  SPIFFS_LOCK(fs);
+
+#if SPIFFS_CACHE
+  spiffs_fflush_cache(fs, fh);
+#endif
+  spiffs_fd_return(fs, fh);
+
+  SPIFFS_UNLOCK(fs);
+}
+
+s32_t SPIFFS_rename(spiffs *fs, const char *old, const char *newname) {
+  SPIFFS_API_CHECK_MOUNT(fs);
+  SPIFFS_LOCK(fs);
+
+  spiffs_page_ix pix_old, pix_dummy;
+  spiffs_fd *fd;
+
+  s32_t res = spiffs_object_find_object_index_header_by_name(fs, (u8_t*)old, &pix_old);
+  SPIFFS_API_CHECK_RES_UNLOCK(fs, res);
+
+  res = spiffs_object_find_object_index_header_by_name(fs, (u8_t*)newname, &pix_dummy);
+  if (res == SPIFFS_ERR_NOT_FOUND) {
+    res = SPIFFS_OK;
+  } else if (res == SPIFFS_OK) {
+    res = SPIFFS_ERR_CONFLICTING_NAME;
+  }
+  SPIFFS_API_CHECK_RES_UNLOCK(fs, res);
+
+  res = spiffs_fd_find_new(fs, &fd);
+  SPIFFS_API_CHECK_RES_UNLOCK(fs, res);
+
+  res = spiffs_object_open_by_page(fs, pix_old, fd, 0, 0);
+  if (res != SPIFFS_OK) {
+    spiffs_fd_return(fs, fd->file_nbr);
+  }
+  SPIFFS_API_CHECK_RES_UNLOCK(fs, res);
+
+  res = spiffs_object_update_index_hdr(fs, fd, fd->obj_id, fd->objix_hdr_pix, 0, (u8_t*)newname,
+      0, &pix_dummy);
+
+  if (res != SPIFFS_OK) {
+    spiffs_fd_return(fs, fd->file_nbr);
+  }
+  SPIFFS_API_CHECK_RES_UNLOCK(fs, res);
+
+  SPIFFS_UNLOCK(fs);
+
+  return res;
+}
+
+spiffs_DIR *SPIFFS_opendir(spiffs *fs, const char *name, spiffs_DIR *d) {
+  (void)name;
+  if (!SPIFFS_CHECK_MOUNT(fs)) {
+    fs->errno = SPIFFS_ERR_NOT_MOUNTED;
+    return 0;
+  }
+  d->fs = fs;
+  d->block = 0;
+  d->entry = 0;
+  return d;
+}
+
+static s32_t spiffs_read_dir_v(
+    spiffs *fs,
+    spiffs_obj_id obj_id,
+    spiffs_block_ix bix,
+    int ix_entry,
+    u32_t user_data,
+    void *user_p) {
+  (void)user_data;
+  s32_t res;
+  spiffs_page_object_ix_header objix_hdr;
+  if (obj_id == SPIFFS_OBJ_ID_FREE || obj_id == SPIFFS_OBJ_ID_DELETED ||
+      (obj_id & SPIFFS_OBJ_ID_IX_FLAG) == 0) {
+    return SPIFFS_VIS_COUNTINUE;
+  }
+
+  spiffs_page_ix pix = SPIFFS_OBJ_LOOKUP_ENTRY_TO_PIX(fs, bix, ix_entry);
+  res = _spiffs_rd(fs, SPIFFS_OP_T_OBJ_LU2 | SPIFFS_OP_C_READ,
+      0, SPIFFS_PAGE_TO_PADDR(fs, pix), sizeof(spiffs_page_object_ix_header), (u8_t *)&objix_hdr);
+  if (res != SPIFFS_OK) return res;
+  if ((obj_id & SPIFFS_OBJ_ID_IX_FLAG) &&
+      objix_hdr.p_hdr.span_ix == 0 &&
+      (objix_hdr.p_hdr.flags& (SPIFFS_PH_FLAG_DELET | SPIFFS_PH_FLAG_FINAL | SPIFFS_PH_FLAG_IXDELE)) ==
+          (SPIFFS_PH_FLAG_DELET | SPIFFS_PH_FLAG_IXDELE)) {
+    struct spiffs_dirent *e = (struct spiffs_dirent *)user_p;
+    e->obj_id = obj_id;
+    strcpy((char *)e->name, (char *)objix_hdr.name);
+    e->type = objix_hdr.type;
+    e->size = objix_hdr.size == SPIFFS_UNDEFINED_LEN ? 0 : objix_hdr.size;
+    e->pix = pix;
+    return SPIFFS_OK;
+  }
+
+  return SPIFFS_VIS_COUNTINUE;
+}
+
+struct spiffs_dirent *SPIFFS_readdir(spiffs_DIR *d, struct spiffs_dirent *e) {
+  if (!SPIFFS_CHECK_MOUNT(d->fs)) {
+    d->fs->errno = SPIFFS_ERR_NOT_MOUNTED;
+    return 0;
+  }
+  SPIFFS_LOCK(fs);
+
+  spiffs_block_ix bix;
+  int entry;
+  s32_t res;
+  struct spiffs_dirent *ret = 0;
+
+  res = spiffs_obj_lu_find_entry_visitor(d->fs,
+      d->block,
+      d->entry,
+      SPIFFS_VIS_NO_WRAP,
+      0,
+      spiffs_read_dir_v,
+      0,
+      e,
+      &bix,
+      &entry);
+  if (res == SPIFFS_OK) {
+    d->block = bix;
+    d->entry = entry + 1;
+    ret = e;
+  } else {
+    d->fs->errno = res;
+  }
+  SPIFFS_UNLOCK(fs);
+  return ret;
+}
+
+s32_t SPIFFS_closedir(spiffs_DIR *d) {
+  SPIFFS_API_CHECK_MOUNT(d->fs);
+  return 0;
+}
+
+s32_t SPIFFS_check(spiffs *fs) {
+  s32_t res;
+  SPIFFS_API_CHECK_MOUNT(fs);
+  SPIFFS_LOCK(fs);
+
+  res = spiffs_lookup_consistency_check(fs, 0);
+
+  res = spiffs_object_index_consistency_check(fs);
+
+  res = spiffs_page_consistency_check(fs);
+
+  res = spiffs_obj_lu_scan(fs);
+
+  SPIFFS_UNLOCK(fs);
+  return res;
+}
+
+s32_t SPIFFS_eof(spiffs *fs, spiffs_file fh) {
+  SPIFFS_API_CHECK_MOUNT(fs);
+  SPIFFS_LOCK(fs);
+
+  spiffs_fd *fd;
+  s32_t res;
+  res = spiffs_fd_get(fs, fh, &fd);
+  SPIFFS_API_CHECK_RES(fs, res);
+
+#if SPIFFS_CACHE_WR
+  spiffs_fflush_cache(fs, fh);
+#endif
+
+  res = (fd->fdoffset == fd->size);
+
+  SPIFFS_UNLOCK(fs);
+  return res;
+}
+
+s32_t SPIFFS_tell(spiffs *fs, spiffs_file fh) {
+  SPIFFS_API_CHECK_MOUNT(fs);
+  SPIFFS_LOCK(fs);
+
+  spiffs_fd *fd;
+  s32_t res;
+  res = spiffs_fd_get(fs, fh, &fd);
+  SPIFFS_API_CHECK_RES(fs, res);
+
+#if SPIFFS_CACHE_WR
+  spiffs_fflush_cache(fs, fh);
+#endif
+
+  res = fd->fdoffset;
+
+  SPIFFS_UNLOCK(fs);
+  return res;
+}
+
+#if SPIFFS_TEST_VISUALISATION
+s32_t SPIFFS_vis(spiffs *fs) {
+  s32_t res = SPIFFS_OK;
+  SPIFFS_API_CHECK_MOUNT(fs);
+  SPIFFS_LOCK(fs);
+
+  int entries_per_page = (SPIFFS_CFG_LOG_PAGE_SZ(fs) / sizeof(spiffs_obj_id));
+  spiffs_obj_id *obj_lu_buf = (spiffs_obj_id *)fs->lu_work;
+  spiffs_block_ix bix = 0;
+
+  while (bix < fs->block_count) {
+    // check each object lookup page
+    int obj_lookup_page = 0;
+    int cur_entry = 0;
+
+    while (res == SPIFFS_OK && obj_lookup_page < (int)SPIFFS_OBJ_LOOKUP_PAGES(fs)) {
+      int entry_offset = obj_lookup_page * entries_per_page;
+      res = _spiffs_rd(fs, SPIFFS_OP_T_OBJ_LU | SPIFFS_OP_C_READ,
+          0, bix * SPIFFS_CFG_LOG_BLOCK_SZ(fs) + SPIFFS_PAGE_TO_PADDR(fs, obj_lookup_page), SPIFFS_CFG_LOG_PAGE_SZ(fs), fs->lu_work);
+      // check each entry
+      while (res == SPIFFS_OK &&
+          cur_entry - entry_offset < entries_per_page && cur_entry < (int)(SPIFFS_PAGES_PER_BLOCK(fs)-SPIFFS_OBJ_LOOKUP_PAGES(fs))) {
+        spiffs_obj_id obj_id = obj_lu_buf[cur_entry-entry_offset];
+        if (cur_entry == 0) {
+          spiffs_printf("%4i ", bix);
+        } else if ((cur_entry & 0x3f) == 0) {
+          spiffs_printf("     ");
+        }
+        if (obj_id == SPIFFS_OBJ_ID_FREE) {
+          spiffs_printf(SPIFFS_TEST_VIS_FREE_STR);
+        } else if (obj_id == SPIFFS_OBJ_ID_DELETED) {
+          spiffs_printf(SPIFFS_TEST_VIS_DELE_STR);
+        } else if (obj_id & SPIFFS_OBJ_ID_IX_FLAG){
+          spiffs_printf(SPIFFS_TEST_VIS_INDX_STR(obj_id));
+        } else {
+          spiffs_printf(SPIFFS_TEST_VIS_DATA_STR(obj_id));
+        }
+        cur_entry++;
+        if ((cur_entry & 0x3f) == 0) {
+          spiffs_printf("\n");
+        }
+      } // per entry
+      obj_lookup_page++;
+    } // per object lookup page
+
+    spiffs_obj_id erase_count;
+    res = _spiffs_rd(fs, SPIFFS_OP_C_READ | SPIFFS_OP_T_OBJ_LU2, 0,
+        SPIFFS_ERASE_COUNT_PADDR(fs, bix),
+        sizeof(spiffs_obj_id), (u8_t *)&erase_count);
+    SPIFFS_CHECK_RES(res);
+
+    if (erase_count != (spiffs_obj_id)-1) {
+      spiffs_printf("\tera_cnt: %d\n", erase_count);
+    } else {
+      spiffs_printf("\tera_cnt: N/A\n");
+    }
+
+    bix++;
+  } // per block
+
+  spiffs_printf("era_cnt_max: %d\n", fs->max_erase_count);
+  spiffs_printf("last_errno:  %d\n", fs->errno);
+  spiffs_printf("blocks:      %d\n", fs->block_count);
+  spiffs_printf("free_blocks: %d\n", fs->free_blocks);
+  spiffs_printf("page_alloc:  %d\n", fs->stats_p_allocated);
+  spiffs_printf("page_delet:  %d\n", fs->stats_p_deleted);
+
+  SPIFFS_UNLOCK(fs);
+  return res;
+}
+#endif

cores/esp8266/spiffs/spiffs_nucleus.h

@@ -0,0 +1,687 @@
+/*
+ * spiffs_nucleus.h
+ *
+ *  Created on: Jun 15, 2013
+ *      Author: petera
+ */
+
+/* SPIFFS layout
+ *
+ * spiffs is designed for following spi flash characteristics:
+ *   - only big areas of data (blocks) can be erased
+ *   - erasing resets all bits in a block to ones
+ *   - writing pulls ones to zeroes
+ *   - zeroes cannot be pulled to ones, without erase
+ *   - wear leveling
+ *
+ * spiffs is also meant to be run on embedded, memory constraint devices.
+ *
+ * Entire area is divided in blocks. Entire area is also divided in pages.
+ * Each block contains same number of pages. A page cannot be erased, but a
+ * block can be erased.
+ *
+ * Entire area must be block_size * x
+ * page_size must be block_size / (2^y) where y > 2
+ *
+ * ex: area = 1024*1024 bytes, block size = 65536 bytes, page size = 256 bytes
+ *
+ * BLOCK 0  PAGE 0       object lookup 1
+ *          PAGE 1       object lookup 2
+ *          ...
+ *          PAGE n-1     object lookup n
+ *          PAGE n       object data 1
+ *          PAGE n+1     object data 2
+ *          ...
+ *          PAGE n+m-1   object data m
+ *
+ * BLOCK 1  PAGE n+m     object lookup 1
+ *          PAGE n+m+1   object lookup 2
+ *          ...
+ *          PAGE 2n+m-1  object lookup n
+ *          PAGE 2n+m    object data 1
+ *          PAGE 2n+m    object data 2
+ *          ...
+ *          PAGE 2n+2m-1 object data m
+ * ...
+ *
+ * n is number of object lookup pages, which is number of pages needed to index all pages
+ * in a block by object id
+ *   : block_size / page_size * sizeof(obj_id) / page_size
+ * m is number data pages, which is number of pages in block minus number of lookup pages
+ *   : block_size / page_size - block_size / page_size * sizeof(obj_id) / page_size
+ * thus, n+m is total number of pages in a block
+ *   : block_size / page_size
+ *
+ * ex: n = 65536/256*2/256 = 2, m = 65536/256 - 2 = 254 => n+m = 65536/256 = 256
+ *
+ * Object lookup pages contain object id entries. Each entry represent the corresponding
+ * data page.
+ * Assuming a 16 bit object id, an object id being 0xffff represents a free page.
+ * An object id being 0x0000 represents a deleted page.
+ *
+ * ex: page 0 : lookup : 0008 0001 0aaa ffff ffff ffff ffff ffff ..
+ *     page 1 : lookup : ffff ffff ffff ffff ffff ffff ffff ffff ..
+ *     page 2 : data   : data for object id 0008
+ *     page 3 : data   : data for object id 0001
+ *     page 4 : data   : data for object id 0aaa
+ *     ...
+ *
+ *
+ * Object data pages can be either object index pages or object content.
+ * All object data pages contains a data page header, containing object id and span index.
+ * The span index denotes the object page ordering amongst data pages with same object id.
+ * This applies to both object index pages (when index spans more than one page of entries),
+ * and object data pages.
+ * An object index page contains page entries pointing to object content page. The entry index
+ * in a object index page correlates to the span index in the actual object data page.
+ * The first object index page (span index 0) is called object index header page, and also
+ * contains object flags (directory/file), size, object name etc.
+ *
+ * ex:
+ *  BLOCK 1
+ *    PAGE 256: objectl lookup page 1
+ *      [*123] [ 123] [ 123] [ 123]
+ *      [ 123] [*123] [ 123] [ 123]
+ *      [free] [free] [free] [free] ...
+ *    PAGE 257: objectl lookup page 2
+ *      [free] [free] [free] [free] ...
+ *    PAGE 258: object index page (header)
+ *      obj.id:0123 span.ix:0000 flags:INDEX
+ *      size:1600 name:ex.txt type:file
+ *      [259] [260] [261] [262]
+ *    PAGE 259: object data page
+ *      obj.id:0123 span.ix:0000 flags:DATA
+ *    PAGE 260: object data page
+ *      obj.id:0123 span.ix:0001 flags:DATA
+ *    PAGE 261: object data page
+ *      obj.id:0123 span.ix:0002 flags:DATA
+ *    PAGE 262: object data page
+ *      obj.id:0123 span.ix:0003 flags:DATA
+ *    PAGE 263: object index page
+ *      obj.id:0123 span.ix:0001 flags:INDEX
+ *      [264] [265] [fre] [fre]
+ *      [fre] [fre] [fre] [fre]
+ *    PAGE 264: object data page
+ *      obj.id:0123 span.ix:0004 flags:DATA
+ *    PAGE 265: object data page
+ *      obj.id:0123 span.ix:0005 flags:DATA
+ *
+ */
+#ifndef SPIFFS_NUCLEUS_H_
+#define SPIFFS_NUCLEUS_H_
+
+#define _SPIFFS_ERR_CHECK_FIRST         (SPIFFS_ERR_INTERNAL - 1)
+#define SPIFFS_ERR_CHECK_OBJ_ID_MISM    (SPIFFS_ERR_INTERNAL - 1)
+#define SPIFFS_ERR_CHECK_SPIX_MISM      (SPIFFS_ERR_INTERNAL - 2)
+#define SPIFFS_ERR_CHECK_FLAGS_BAD      (SPIFFS_ERR_INTERNAL - 3)
+#define _SPIFFS_ERR_CHECK_LAST          (SPIFFS_ERR_INTERNAL - 4)
+
+#define SPIFFS_VIS_COUNTINUE            (SPIFFS_ERR_INTERNAL - 20)
+#define SPIFFS_VIS_COUNTINUE_RELOAD     (SPIFFS_ERR_INTERNAL - 21)
+#define SPIFFS_VIS_END                  (SPIFFS_ERR_INTERNAL - 22)
+
+#define SPIFFS_EV_IX_UPD                0
+#define SPIFFS_EV_IX_NEW                1
+#define SPIFFS_EV_IX_DEL                2
+
+#define SPIFFS_OBJ_ID_IX_FLAG           (1<<(8*sizeof(spiffs_obj_id)-1))
+
+#define SPIFFS_UNDEFINED_LEN            (u32_t)(-1)
+
+#define SPIFFS_OBJ_ID_DELETED           ((spiffs_obj_id)0)
+#define SPIFFS_OBJ_ID_FREE              ((spiffs_obj_id)-1)
+
+#if SPIFFS_SINGLETON == 0
+#define SPIFFS_CFG_LOG_PAGE_SZ(fs) \
+  ((fs)->cfg.log_page_size)
+#define SPIFFS_CFG_LOG_BLOCK_SZ(fs) \
+  ((fs)->cfg.log_block_size)
+#define SPIFFS_CFG_PHYS_SZ(fs) \
+  ((fs)->cfg.phys_size)
+#define SPIFFS_CFG_PHYS_ERASE_SZ(fs) \
+  ((fs)->cfg.phys_erase_block)
+#define SPIFFS_CFG_PHYS_ADDR(fs) \
+  ((fs)->cfg.phys_addr)
+#endif
+
+// total number of pages
+#define SPIFFS_MAX_PAGES(fs) \
+  ( SPIFFS_CFG_PHYS_SZ(fs)/SPIFFS_CFG_LOG_PAGE_SZ(fs) )
+// total number of pages per block, including object lookup pages
+#define SPIFFS_PAGES_PER_BLOCK(fs) \
+  ( SPIFFS_CFG_LOG_BLOCK_SZ(fs)/SPIFFS_CFG_LOG_PAGE_SZ(fs) )
+// number of object lookup pages per block
+#define SPIFFS_OBJ_LOOKUP_PAGES(fs)     \
+  (MAX(1, (SPIFFS_PAGES_PER_BLOCK(fs) * sizeof(spiffs_obj_id)) / SPIFFS_CFG_LOG_PAGE_SZ(fs)) )
+// checks if page index belongs to object lookup
+#define SPIFFS_IS_LOOKUP_PAGE(fs,pix)     \
+  (((pix) % SPIFFS_PAGES_PER_BLOCK(fs)) < SPIFFS_OBJ_LOOKUP_PAGES(fs))
+// number of object lookup entries in all object lookup pages
+#define SPIFFS_OBJ_LOOKUP_MAX_ENTRIES(fs) \
+  (SPIFFS_PAGES_PER_BLOCK(fs)-SPIFFS_OBJ_LOOKUP_PAGES(fs))
+// converts a block to physical address
+#define SPIFFS_BLOCK_TO_PADDR(fs, block) \
+  ( SPIFFS_CFG_PHYS_ADDR(fs) + (block)* SPIFFS_CFG_LOG_BLOCK_SZ(fs) )
+// converts a object lookup entry to page index
+#define SPIFFS_OBJ_LOOKUP_ENTRY_TO_PIX(fs, block, entry) \
+  ((block)*SPIFFS_PAGES_PER_BLOCK(fs) + (SPIFFS_OBJ_LOOKUP_PAGES(fs) + entry))
+// converts a object lookup entry to physical address of corresponding page
+#define SPIFFS_OBJ_LOOKUP_ENTRY_TO_PADDR(fs, block, entry) \
+  (SPIFFS_BLOCK_TO_PADDR(fs, block) + (SPIFFS_OBJ_LOOKUP_PAGES(fs) + entry) * SPIFFS_CFG_LOG_PAGE_SZ(fs) )
+// converts a page to physical address
+#define SPIFFS_PAGE_TO_PADDR(fs, page) \
+  ( SPIFFS_CFG_PHYS_ADDR(fs) + (page) * SPIFFS_CFG_LOG_PAGE_SZ(fs) )
+// converts a physical address to page
+#define SPIFFS_PADDR_TO_PAGE(fs, addr) \
+  ( ((addr) -  SPIFFS_CFG_PHYS_ADDR(fs)) / SPIFFS_CFG_LOG_PAGE_SZ(fs) )
+// gives index in page for a physical address
+#define SPIFFS_PADDR_TO_PAGE_OFFSET(fs, addr) \
+  ( ((addr) - SPIFFS_CFG_PHYS_ADDR(fs)) % SPIFFS_CFG_LOG_PAGE_SZ(fs) )
+// returns containing block for given page
+#define SPIFFS_BLOCK_FOR_PAGE(fs, page) \
+  ( (page) / SPIFFS_PAGES_PER_BLOCK(fs) )
+// returns starting page for block
+#define SPIFFS_PAGE_FOR_BLOCK(fs, block) \
+  ( (block) * SPIFFS_PAGES_PER_BLOCK(fs) )
+// converts page to entry in object lookup page
+#define SPIFFS_OBJ_LOOKUP_ENTRY_FOR_PAGE(fs, page) \
+  ( (page) % SPIFFS_PAGES_PER_BLOCK(fs) - SPIFFS_OBJ_LOOKUP_PAGES(fs) )
+// returns data size in a data page
+#define SPIFFS_DATA_PAGE_SIZE(fs) \
+    ( SPIFFS_CFG_LOG_PAGE_SZ(fs) - sizeof(spiffs_page_header) )
+// returns physical address for block's erase count
+#define SPIFFS_ERASE_COUNT_PADDR(fs, bix) \
+  ( SPIFFS_BLOCK_TO_PADDR(fs, bix) + SPIFFS_OBJ_LOOKUP_PAGES(fs) * SPIFFS_CFG_LOG_PAGE_SZ(fs) - sizeof(spiffs_obj_id) )
+
+// define helpers object
+
+// entries in an object header page index
+#define SPIFFS_OBJ_HDR_IX_LEN(fs) \
+  ((SPIFFS_CFG_LOG_PAGE_SZ(fs) - sizeof(spiffs_page_object_ix_header))/sizeof(spiffs_page_ix))
+// entries in an object page index
+#define SPIFFS_OBJ_IX_LEN(fs) \
+  ((SPIFFS_CFG_LOG_PAGE_SZ(fs) - sizeof(spiffs_page_object_ix))/sizeof(spiffs_page_ix))
+// object index entry for given data span index
+#define SPIFFS_OBJ_IX_ENTRY(fs, spix) \
+  ((spix) < SPIFFS_OBJ_HDR_IX_LEN(fs) ? (spix) : (((spix)-SPIFFS_OBJ_HDR_IX_LEN(fs))%SPIFFS_OBJ_IX_LEN(fs)))
+// object index span index number for given data span index or entry
+#define SPIFFS_OBJ_IX_ENTRY_SPAN_IX(fs, spix) \
+  ((spix) < SPIFFS_OBJ_HDR_IX_LEN(fs) ? 0 : (1+((spix)-SPIFFS_OBJ_HDR_IX_LEN(fs))/SPIFFS_OBJ_IX_LEN(fs)))
+
+
+#define SPIFFS_OP_T_OBJ_LU    (0<<0)
+#define SPIFFS_OP_T_OBJ_LU2   (1<<0)
+#define SPIFFS_OP_T_OBJ_IX    (2<<0)
+#define SPIFFS_OP_T_OBJ_DA    (3<<0)
+#define SPIFFS_OP_C_DELE      (0<<2)
+#define SPIFFS_OP_C_UPDT      (1<<2)
+#define SPIFFS_OP_C_MOVS      (2<<2)
+#define SPIFFS_OP_C_MOVD      (3<<2)
+#define SPIFFS_OP_C_FLSH      (4<<2)
+#define SPIFFS_OP_C_READ      (5<<2)
+#define SPIFFS_OP_C_WRTHRU    (6<<2)
+
+#define SPIFFS_OP_TYPE_MASK (3<<0)
+#define SPIFFS_OP_COM_MASK  (7<<2)
+
+
+// if 0, this page is written to, else clean
+#define SPIFFS_PH_FLAG_USED   (1<<0)
+// if 0, writing is finalized, else under modification
+#define SPIFFS_PH_FLAG_FINAL  (1<<1)
+// if 0, this is an index page, else a data page
+#define SPIFFS_PH_FLAG_INDEX  (1<<2)
+// if 0, page is deleted, else valid
+#define SPIFFS_PH_FLAG_DELET  (1<<7)
+// if 0, this index header is being deleted
+#define SPIFFS_PH_FLAG_IXDELE (1<<6)
+
+
+#define SPIFFS_CHECK_MOUNT(fs) \
+  ((fs)->block_count > 0)
+
+#define SPIFFS_CHECK_RES(res) \
+  do { \
+    if ((res) < SPIFFS_OK) return (res); \
+  } while (0);
+
+#define SPIFFS_API_CHECK_MOUNT(fs) \
+  if (!SPIFFS_CHECK_MOUNT((fs))) { \
+    (fs)->errno = SPIFFS_ERR_NOT_MOUNTED; \
+    return -1; \
+  }
+
+#define SPIFFS_API_CHECK_RES(fs, res) \
+  if ((res) < SPIFFS_OK) { \
+    (fs)->errno = (res); \
+    return -1; \
+  }
+
+#define SPIFFS_API_CHECK_RES_UNLOCK(fs, res) \
+  if ((res) < SPIFFS_OK) { \
+    (fs)->errno = (res); \
+    SPIFFS_UNLOCK(fs); \
+    return -1; \
+  }
+
+#define SPIFFS_VALIDATE_OBJIX(ph, objid, spix) \
+    if (((ph).flags & SPIFFS_PH_FLAG_USED) != 0) return SPIFFS_ERR_IS_FREE; \
+    if (((ph).flags & SPIFFS_PH_FLAG_DELET) == 0) return SPIFFS_ERR_DELETED; \
+    if (((ph).flags & SPIFFS_PH_FLAG_FINAL) != 0) return SPIFFS_ERR_NOT_FINALIZED; \
+    if (((ph).flags & SPIFFS_PH_FLAG_INDEX) != 0) return SPIFFS_ERR_NOT_INDEX; \
+    if (((objid) & SPIFFS_OBJ_ID_IX_FLAG) == 0) return SPIFFS_ERR_NOT_INDEX; \
+    if ((ph).span_ix != (spix)) return SPIFFS_ERR_INDEX_SPAN_MISMATCH;
+    //if ((spix) == 0 && ((ph).flags & SPIFFS_PH_FLAG_IXDELE) == 0) return SPIFFS_ERR_DELETED;
+
+#define SPIFFS_VALIDATE_DATA(ph, objid, spix) \
+    if (((ph).flags & SPIFFS_PH_FLAG_USED) != 0) return SPIFFS_ERR_IS_FREE; \
+    if (((ph).flags & SPIFFS_PH_FLAG_DELET) == 0) return SPIFFS_ERR_DELETED; \
+    if (((ph).flags & SPIFFS_PH_FLAG_FINAL) != 0) return SPIFFS_ERR_NOT_FINALIZED; \
+    if (((ph).flags & SPIFFS_PH_FLAG_INDEX) == 0) return SPIFFS_ERR_IS_INDEX; \
+    if ((objid) & SPIFFS_OBJ_ID_IX_FLAG) return SPIFFS_ERR_IS_INDEX; \
+    if ((ph).span_ix != (spix)) return SPIFFS_ERR_DATA_SPAN_MISMATCH;
+
+
+// check id
+#define SPIFFS_VIS_CHECK_ID     (1<<0)
+// report argument object id to visitor - else object lookup id is reported
+#define SPIFFS_VIS_CHECK_PH     (1<<1)
+// stop searching at end of all look up pages
+#define SPIFFS_VIS_NO_WRAP      (1<<2)
+
+#if SPIFFS_CACHE
+
+#define SPIFFS_CACHE_FLAG_DIRTY       (1<<0)
+#define SPIFFS_CACHE_FLAG_WRTHRU      (1<<1)
+#define SPIFFS_CACHE_FLAG_OBJLU       (1<<2)
+#define SPIFFS_CACHE_FLAG_OBJIX       (1<<3)
+#define SPIFFS_CACHE_FLAG_DATA        (1<<4)
+#define SPIFFS_CACHE_FLAG_TYPE_WR     (1<<7)
+
+#define SPIFFS_CACHE_PAGE_SIZE(fs) \
+  (sizeof(spiffs_cache_page) + SPIFFS_CFG_LOG_PAGE_SZ(fs))
+
+#define spiffs_get_cache(fs) \
+  ((spiffs_cache *)((fs)->cache))
+
+#define spiffs_get_cache_page_hdr(fs, c, ix) \
+  ((spiffs_cache_page *)(&((c)->cpages[(ix) * SPIFFS_CACHE_PAGE_SIZE(fs)])))
+
+#define spiffs_get_cache_page(fs, c, ix) \
+  ((u8_t *)(&((c)->cpages[(ix) * SPIFFS_CACHE_PAGE_SIZE(fs)])) + sizeof(spiffs_cache_page))
+
+// cache page struct
+typedef struct {
+  // cache flags
+  u8_t flags;
+  // cache page index
+  u8_t ix;
+  // last access of this cache page
+  u32_t last_access;
+  union {
+    // type read cache
+    struct {
+      // read cache page index
+      spiffs_page_ix pix;
+    };
+#if SPIFFS_CACHE_WR
+    // type write cache
+    struct {
+      // write cache
+      spiffs_obj_id obj_id;
+      // offset in cache page
+      u32_t offset;
+      // size of cache page
+      u16_t size;
+    };
+#endif
+  };
+} spiffs_cache_page;
+
+// cache struct
+typedef struct {
+  u8_t cpage_count;
+  u32_t last_access;
+  u32_t cpage_use_map;
+  u32_t cpage_use_mask;
+  u8_t *cpages;
+} spiffs_cache;
+
+#endif
+
+
+// spiffs nucleus file descriptor
+typedef struct {
+  // the filesystem of this descriptor
+  spiffs *fs;
+  // number of file descriptor - if 0, the file descriptor is closed
+  spiffs_file file_nbr;
+  // object id - if SPIFFS_OBJ_ID_ERASED, the file was deleted
+  spiffs_obj_id obj_id;
+  // size of the file
+  u32_t size;
+  // cached object index header page index
+  spiffs_page_ix objix_hdr_pix;
+  // cached offset object index page index
+  spiffs_page_ix cursor_objix_pix;
+  // cached offset object index span index
+  spiffs_span_ix cursor_objix_spix;
+  // current absolute offset
+  u32_t offset;
+  // current file descriptor offset
+  u32_t fdoffset;
+  // fd flags
+  spiffs_flags flags;
+#if SPIFFS_CACHE_WR
+  spiffs_cache_page *cache_page;
+#endif
+} spiffs_fd;
+
+
+// object structs
+
+// page header, part of each page except object lookup pages
+typedef struct __attribute(( packed )) {
+  // object id
+  spiffs_obj_id obj_id;
+  // object span index
+  spiffs_span_ix span_ix;
+  // flags
+  u8_t flags;
+} spiffs_page_header;
+
+// object index header page header
+typedef struct __attribute(( packed )) {
+  // common page header
+  spiffs_page_header p_hdr;
+  // alignment
+  u8_t _align[4 - (sizeof(spiffs_page_header)&3)==0 ? 4 : (sizeof(spiffs_page_header)&3)];
+  // size of object
+  u32_t size;
+  // type of object
+  spiffs_obj_type type;
+  // alignment2
+  u8_t _align2[4 - (sizeof(spiffs_obj_type)&3)==0 ? 4 : (sizeof(spiffs_obj_type)&3)];
+  // name of object
+  u8_t name[SPIFFS_OBJ_NAME_LEN];
+} spiffs_page_object_ix_header;
+
+// object index page header
+typedef struct __attribute(( packed )) {
+ spiffs_page_header p_hdr;
+ u8_t _align[4 - (sizeof(spiffs_page_header)&3)==0 ? 4 : (sizeof(spiffs_page_header)&3)];
+} spiffs_page_object_ix;
+
+// callback func for object lookup visitor
+typedef s32_t (*spiffs_visitor_f)(spiffs *fs, spiffs_obj_id id, spiffs_block_ix bix, int ix_entry,
+    u32_t user_data, void *user_p);
+
+
+#if SPIFFS_CACHE
+#define _spiffs_rd(fs, op, fh, addr, len, dst) \
+    spiffs_phys_rd((fs), (op), (fh), (addr), (len), (dst))
+#define _spiffs_wr(fs, op, fh, addr, len, src) \
+    spiffs_phys_wr((fs), (op), (fh), (addr), (len), (src))
+#else
+#define _spiffs_rd(fs, op, fh, addr, len, dst) \
+    spiffs_phys_rd((fs), (addr), (len), (dst))
+#define _spiffs_wr(fs, op, fh, addr, len, src) \
+    spiffs_phys_wr((fs), (addr), (len), (src))
+#endif
+
+#ifndef MIN
+#define MIN(a,b) ((a) < (b) ? (a) : (b))
+#endif
+#ifndef MAX
+#define MAX(a,b) ((a) > (b) ? (a) : (b))
+#endif
+
+// ---------------
+
+s32_t spiffs_phys_rd(
+    spiffs *fs,
+#if SPIFFS_CACHE
+    u8_t op,
+    spiffs_file fh,
+#endif
+    u32_t addr,
+    u32_t len,
+    u8_t *dst);
+
+s32_t spiffs_phys_wr(
+    spiffs *fs,
+#if SPIFFS_CACHE
+    u8_t op,
+    spiffs_file fh,
+#endif
+    u32_t addr,
+    u32_t len,
+    u8_t *src);
+
+s32_t spiffs_phys_cpy(
+    spiffs *fs,
+    spiffs_file fh,
+    u32_t dst,
+    u32_t src,
+    u32_t len);
+
+s32_t spiffs_phys_count_free_blocks(
+    spiffs *fs);
+
+s32_t spiffs_obj_lu_find_entry_visitor(
+    spiffs *fs,
+    spiffs_block_ix starting_block,
+    int starting_lu_entry,
+    u8_t flags,
+    spiffs_obj_id obj_id,
+    spiffs_visitor_f v,
+    u32_t user_data,
+    void *user_p,
+    spiffs_block_ix *block_ix,
+    int *lu_entry);
+
+// ---------------
+
+s32_t spiffs_obj_lu_scan(
+    spiffs *fs);
+
+s32_t spiffs_obj_lu_find_free_obj_id(
+    spiffs *fs,
+    spiffs_obj_id *obj_id,
+    u8_t *conflicting_name);
+
+s32_t spiffs_obj_lu_find_free(
+    spiffs *fs,
+    spiffs_block_ix starting_block,
+    int starting_lu_entry,
+    spiffs_block_ix *block_ix,
+    int *lu_entry);
+
+s32_t spiffs_obj_lu_find_id(
+    spiffs *fs,
+    spiffs_block_ix starting_block,
+    int starting_lu_entry,
+    spiffs_obj_id obj_id,
+    spiffs_block_ix *block_ix,
+    int *lu_entry);
+
+s32_t spiffs_obj_lu_find_id_and_span(
+    spiffs *fs,
+    spiffs_obj_id obj_id,
+    spiffs_span_ix spix,
+    spiffs_page_ix exclusion_pix,
+    spiffs_page_ix *pix);
+
+s32_t spiffs_obj_lu_find_id_and_span_by_phdr(
+    spiffs *fs,
+    spiffs_obj_id obj_id,
+    spiffs_span_ix spix,
+    spiffs_page_ix exclusion_pix,
+    spiffs_page_ix *pix);
+
+// ---------------
+
+s32_t spiffs_page_allocate_data(
+    spiffs *fs,
+    spiffs_obj_id obj_id,
+    spiffs_page_header *ph,
+    u8_t *data,
+    u32_t len,
+    u32_t page_offs,
+    u8_t finalize,
+    spiffs_page_ix *pix);
+
+s32_t spiffs_page_move(
+    spiffs *fs,
+    spiffs_file fh,
+    u8_t *page_data,
+    spiffs_obj_id obj_id,
+    spiffs_page_header *page_hdr,
+    spiffs_page_ix src_pix,
+    spiffs_page_ix *dst_pix);
+
+s32_t spiffs_page_delete(
+    spiffs *fs,
+    spiffs_page_ix pix);
+
+// ---------------
+
+s32_t spiffs_object_create(
+    spiffs *fs,
+    spiffs_obj_id obj_id,
+    u8_t name[SPIFFS_OBJ_NAME_LEN],
+    spiffs_obj_type type,
+    spiffs_page_ix *objix_hdr_pix);
+
+s32_t spiffs_object_update_index_hdr(
+    spiffs *fs,
+    spiffs_fd *fd,
+    spiffs_obj_id obj_id,
+    spiffs_page_ix objix_hdr_pix,
+    u8_t *new_objix_hdr_data,
+    u8_t name[SPIFFS_OBJ_NAME_LEN],
+    u32_t size,
+    spiffs_page_ix *new_pix);
+
+void spiffs_cb_object_event(
+    spiffs *fs,
+    spiffs_fd *fd,
+    int ev,
+    spiffs_obj_id obj_id,
+    spiffs_span_ix spix,
+    spiffs_page_ix new_pix,
+    u32_t new_size);
+
+s32_t spiffs_object_open_by_id(
+    spiffs *fs,
+    spiffs_obj_id obj_id,
+    spiffs_fd *f,
+    spiffs_flags flags,
+    spiffs_mode mode);
+
+s32_t spiffs_object_open_by_page(
+    spiffs *fs,
+    spiffs_page_ix pix,
+    spiffs_fd *f,
+    spiffs_flags flags,
+    spiffs_mode mode);
+
+s32_t spiffs_object_append(
+    spiffs_fd *fd,
+    u32_t offset,
+    u8_t *data,
+    u32_t len);
+
+s32_t spiffs_object_modify(
+    spiffs_fd *fd,
+    u32_t offset,
+    u8_t *data,
+    u32_t len);
+
+s32_t spiffs_object_read(
+    spiffs_fd *fd,
+    u32_t offset,
+    u32_t len,
+    u8_t *dst);
+
+s32_t spiffs_object_truncate(
+    spiffs_fd *fd,
+    u32_t new_len,
+    u8_t remove_object);
+
+s32_t spiffs_object_find_object_index_header_by_name(
+    spiffs *fs,
+    u8_t name[SPIFFS_OBJ_NAME_LEN],
+    spiffs_page_ix *pix);
+
+// ---------------
+
+s32_t spiffs_gc_check(
+    spiffs *fs,
+    u32_t len);
+
+s32_t spiffs_gc_erase_page_stats(
+    spiffs *fs,
+    spiffs_block_ix bix);
+
+s32_t spiffs_gc_find_candidate(
+    spiffs *fs,
+    spiffs_block_ix **block_candidate,
+    int *candidate_count);
+
+s32_t spiffs_gc_clean(
+    spiffs *fs,
+    spiffs_block_ix bix);
+
+s32_t spiffs_gc_quick(
+    spiffs *fs);
+
+// ---------------
+
+s32_t spiffs_fd_find_new(
+    spiffs *fs,
+    spiffs_fd **fd);
+
+s32_t spiffs_fd_return(
+    spiffs *fs,
+    spiffs_file f);
+
+s32_t spiffs_fd_get(
+    spiffs *fs,
+    spiffs_file f,
+    spiffs_fd **fd);
+
+#if SPIFFS_CACHE
+void spiffs_cache_init(
+    spiffs *fs);
+
+void spiffs_cache_drop_page(
+    spiffs *fs,
+    spiffs_page_ix pix);
+
+#if SPIFFS_CACHE_WR
+spiffs_cache_page *spiffs_cache_page_allocate_by_fd(
+    spiffs *fs,
+    spiffs_fd *fd);
+
+void spiffs_cache_fd_release(
+    spiffs *fs,
+    spiffs_cache_page *cp);
+
+spiffs_cache_page *spiffs_cache_page_get_by_fd(
+    spiffs *fs,
+    spiffs_fd *fd);
+#endif
+#endif
+
+s32_t spiffs_lookup_consistency_check(
+    spiffs *fs,
+    u8_t check_all_objects);
+
+s32_t spiffs_page_consistency_check(
+    spiffs *fs);
+
+s32_t spiffs_object_index_consistency_check(
+    spiffs *fs);
+
+#endif /* SPIFFS_NUCLEUS_H_ */

libraries/ESP8266WebServer/examples/SDWebServer/SDWebServer.ino

@@ -111,23 +111,10 @@ bool loadFromSdCard(String path){
     dataType = 0;
     path = 0;
     
-    uint8_t obuf[WWW_BUF_SIZE];
+    if(client.write(dataFile) != dataFile.size()){
-    
-    while (dataFile.available() > WWW_BUF_SIZE){
-      dataFile.read(obuf, WWW_BUF_SIZE);
-      if(client.write(obuf, WWW_BUF_SIZE) != WWW_BUF_SIZE){
-        DBG_OUTPUT_PORT.println("Sent less data than expected!");
-        dataFile.close();
-        return true;
-      }
-    }
-    uint16_t leftLen = dataFile.available();
-    dataFile.read(obuf, leftLen);
-    if(client.write(obuf, leftLen) != leftLen){
       DBG_OUTPUT_PORT.println("Sent less data than expected!");
-      dataFile.close();
-      return true;
     }
+    
     dataFile.close();
     client.stop();
     return true;

libraries/ESP8266WebServer/src/ESP8266WebServer.cpp

@@ -327,6 +327,15 @@ void ESP8266WebServer::_parseArguments(String data) {
 
 }
 
+void ESP8266WebServer::_uploadWriteByte(uint8_t b){
+  if(_currentUpload.buflen == HTTP_UPLOAD_BUFLEN){
+    if(_fileUploadHandler) _fileUploadHandler();
+    _currentUpload.size += _currentUpload.buflen;
+    _currentUpload.buflen = 0;
+  }
+  _currentUpload.buf[_currentUpload.buflen++] = b;
+}
+
 void ESP8266WebServer::_parseForm(WiFiClient& client, String boundary, uint32_t len){
   
 #ifdef DEBUG
@@ -428,41 +437,25 @@ void ESP8266WebServer::_parseForm(WiFiClient& client, String boundary, uint32_t
             uint8_t argByte = client.read();
 readfile:
             while(argByte != 0x0D){
-              _currentUpload.buf[_currentUpload.buflen++] = argByte;
+              _uploadWriteByte(argByte);
-              if(_currentUpload.buflen == 1460){
-  #ifdef DEBUG
-                DEBUG_OUTPUT.println("Write File: 1460");
-  #endif
-                if(_fileUploadHandler) _fileUploadHandler();
-                _currentUpload.size += _currentUpload.buflen;
-                _currentUpload.buflen = 0;
-              }
               argByte = client.read();
             }
             
             argByte = client.read();
             if(argByte == 0x0A){
-#ifdef DEBUG
-              DEBUG_OUTPUT.print("Write File: ");
-              DEBUG_OUTPUT.println(_currentUpload.buflen);
-#endif
-              if(_fileUploadHandler) _fileUploadHandler();
-              _currentUpload.size += _currentUpload.buflen;
-              _currentUpload.buflen = 0;
-              
               argByte = client.read();
               if((char)argByte != '-'){
                 //continue reading the file
-                _currentUpload.buf[_currentUpload.buflen++] = 0x0D;
+                _uploadWriteByte(0x0D);
-                _currentUpload.buf[_currentUpload.buflen++] = 0x0A;
+                _uploadWriteByte(0x0A);
                 goto readfile;
               } else {
                 argByte = client.read();
                 if((char)argByte != '-'){
                   //continue reading the file
-                  _currentUpload.buf[_currentUpload.buflen++] = 0x0D;
+                  _uploadWriteByte(0x0D);
-                  _currentUpload.buf[_currentUpload.buflen++] = 0x0A;
+                  _uploadWriteByte(0x0A);
-                  _currentUpload.buf[_currentUpload.buflen++] = (uint8_t)('-');
+                  _uploadWriteByte((uint8_t)('-'));
                   goto readfile;
                 }
               }
@@ -471,7 +464,10 @@ readfile:
               client.readBytes(endBuf, boundary.length());
               
               if(strstr((const char*)endBuf, (const char*)(boundary.c_str())) != NULL){
+                if(_fileUploadHandler) _fileUploadHandler();
+                _currentUpload.size += _currentUpload.buflen;
                 _currentUpload.status = UPLOAD_FILE_END;
+                if(_fileUploadHandler) _fileUploadHandler();
 #ifdef DEBUG
                 DEBUG_OUTPUT.print("End File: ");
                 DEBUG_OUTPUT.print(_currentUpload.filename);
@@ -480,7 +476,6 @@ readfile:
                 DEBUG_OUTPUT.print(" Size: ");
                 DEBUG_OUTPUT.println(_currentUpload.size);
 #endif
-                if(_fileUploadHandler) _fileUploadHandler();
                 line = client.readStringUntil(0x0D);
                 client.readStringUntil(0x0A);
                 if(line == "--"){
@@ -491,33 +486,17 @@ readfile:
                 }
                 continue;
               } else {
-                _currentUpload.buf[_currentUpload.buflen++] = 0x0D;
+                _uploadWriteByte(0x0D);
-                _currentUpload.buf[_currentUpload.buflen++] = 0x0A;
+                _uploadWriteByte(0x0A);
                 uint32_t i = 0;
                 while(i < boundary.length()){
-                  _currentUpload.buf[_currentUpload.buflen++] = endBuf[i++];
+                  _uploadWriteByte(endBuf[i++]);
-                  if(_currentUpload.buflen == 1460){
-#ifdef DEBUG
-                    DEBUG_OUTPUT.println("Write File: 1460");
-#endif
-                    if(_fileUploadHandler) _fileUploadHandler();
-                    _currentUpload.size += _currentUpload.buflen;
-                    _currentUpload.buflen = 0;
-                  }
                 }
                 argByte = client.read();
                 goto readfile;
               }
             } else {
-              _currentUpload.buf[_currentUpload.buflen++] = 0x0D;
+              _uploadWriteByte(0x0D);
-              if(_currentUpload.buflen == 1460){
-  #ifdef DEBUG
-                DEBUG_OUTPUT.println("Write File: 1460");
-  #endif
-                if(_fileUploadHandler) _fileUploadHandler();
-                _currentUpload.size += _currentUpload.buflen;
-                _currentUpload.buflen = 0;
-              }
               goto readfile;
             }
             break;

libraries/ESP8266WebServer/src/ESP8266WebServer.h

@@ -29,6 +29,8 @@
 enum HTTPMethod { HTTP_ANY, HTTP_GET, HTTP_POST, HTTP_PUT, HTTP_PATCH, HTTP_DELETE };
 enum HTTPUploadStatus { UPLOAD_FILE_START, UPLOAD_FILE_WRITE, UPLOAD_FILE_END };
 
+#define HTTP_UPLOAD_BUFLEN 2048
+
 typedef struct {
   HTTPUploadStatus status;
   String filename;
@@ -36,7 +38,7 @@ typedef struct {
   String type;
   size_t size;
   size_t buflen;
-  uint8_t buf[1460];
+  uint8_t buf[HTTP_UPLOAD_BUFLEN];
 } HTTPUpload;
 
 class ESP8266WebServer
@@ -78,6 +80,7 @@ protected:
   static const char* _responseCodeToString(int code);
   static void _appendHeader(String& response, const char* name, const char* value);
   void _parseForm(WiFiClient& client, String boundary, uint32_t len);
+  void _uploadWriteByte(uint8_t b);
   
   struct RequestHandler;
   struct RequestArgument {

libraries/ESP8266WiFi/src/WiFiClient.cpp

@@ -123,6 +123,19 @@ void ICACHE_FLASH_ATTR WiFiClient::_err(int8_t err)
     esp_schedule();
 }
 
+
+void ICACHE_FLASH_ATTR WiFiClient::setNoDelay(bool nodelay) {
+    if (!_client)
+        return;
+    _client->setNoDelay(nodelay);
+}
+
+bool ICACHE_FLASH_ATTR WiFiClient::getNoDelay() {
+    if (!_client)
+        return false;
+    return _client->getNoDelay();
+}
+
 size_t ICACHE_FLASH_ATTR WiFiClient::write(uint8_t b) 
 {
     return write(&b, 1);

libraries/ESP8266WiFi/src/WiFiClient.h

@@ -55,6 +55,30 @@ public:
 
   IPAddress remoteIP();
   uint16_t  remotePort();
+  bool getNoDelay();
+  void setNoDelay(bool nodelay);
+
+  template<typename T> size_t write(T &src){
+    uint8_t obuf[1460];
+    size_t doneLen = 0;
+    size_t sentLen;
+    int i;
+    
+    while (src.available() > 1460){
+      src.read(obuf, 1460);
+      sentLen = write(obuf, 1460);
+      doneLen = doneLen + sentLen;
+      if(sentLen != 1460){
+        return doneLen;
+      }
+    }
+      
+    uint16_t leftLen = src.available();
+    src.read(obuf, leftLen);
+    sentLen = write(obuf, leftLen);
+    doneLen = doneLen + sentLen;
+    return doneLen;
+  }
 
   friend class WiFiServer;