Merge pull request #294 from ficeto/esp8266

Esp8266 SPIFFS Tool

build/shared/tools/ESP8266FS/make.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+ALIBDIR="/Users/ficeto/Documents/Arduino"
+mkdir -p bin && \
+javac -target 1.8 -cp "../../arduino-core.jar:../../pde.jar" -d bin src/ESP8266FS.java && \
+cd bin && \
+mkdir -p $ALIBDIR/tools && \
+rm -rf $ALIBDIR/tools/ESP8266FS && \
+mkdir -p $ALIBDIR/tools/ESP8266FS/tool && \
+zip -r $ALIBDIR/tools/ESP8266FS/tool/esp8266fs.jar * && \
+cd ..

build/shared/tools/ESP8266FS/src/ESP8266FS.java

@@ -0,0 +1,217 @@
+/* -*- mode: java; c-basic-offset: 2; indent-tabs-mode: nil -*- */
+
+/*
+  Tool to put the contents of the sketch's "data" subfolder
+  into an SPIFFS partition image and upload it to an ESP8266 MCU
+
+  Copyright (c) 2015 Hristo Gochkov (ficeto at ficeto dot com)
+
+  This program is free software; you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation; either version 2 of the License, or
+  (at your option) any later version.
+
+  This program is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with this program; if not, write to the Free Software Foundation,
+  Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+*/
+
+package com.esp8266.mkspiffs;
+
+import java.io.File;
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+
+import java.text.SimpleDateFormat;
+import java.util.Date;
+
+import javax.swing.JOptionPane;
+
+import processing.app.PreferencesData;
+import processing.app.Editor;
+import processing.app.Base;
+import processing.app.Platform;
+import processing.app.Sketch;
+import processing.app.tools.Tool;
+import processing.app.helpers.ProcessUtils;
+import processing.app.debug.TargetPlatform;
+
+
+/**
+ * Example Tools menu entry.
+ */
+public class ESP8266FS implements Tool {
+  Editor editor;
+
+
+  public void init(Editor editor) {
+    this.editor = editor;
+  }
+
+
+  public String getMenuTitle() {
+    return "ESP8266 Sketch Data Upload";
+  }
+  
+  private int listenOnProcess(String[] arguments){
+    try {
+      final Process p = ProcessUtils.exec(arguments);
+      Thread thread = new Thread() {
+        public void run() {
+          try {
+            String line;
+            BufferedReader input = new BufferedReader (new InputStreamReader(p.getInputStream()));
+            while ((line = input.readLine()) != null) System.out.println(line);
+            input.close();
+          } catch (Exception e){}
+        }
+      };
+      thread.start();
+      int res = p.waitFor();
+      thread.join();
+      return res;
+    } catch (Exception e){
+      return -1;
+    }
+  }
+  
+  private void sysExec(final String[] arguments){
+    Thread thread = new Thread() {
+      public void run() {
+        try {
+          if(listenOnProcess(arguments) != 0){
+            editor.statusError("SPIFFS Upload failed!");
+          } else {
+            editor.statusNotice("SPIFFS Image Uploaded");
+          }
+        } catch (Exception e){
+          editor.statusError("SPIFFS Upload failed!");
+        }
+      }
+    };
+    thread.start();
+  }
+  
+  
+  private long getIntPref(String name){
+    String data = Base.getBoardPreferences().get(name);
+    if(data == null || data.contentEquals("")) return 0;
+    if(data.startsWith("0x")) return Long.parseLong(data.substring(2), 16);
+    else return Integer.parseInt(data);
+  }
+  
+  private void createAndUpload(){
+    if(!PreferencesData.get("target_platform").contentEquals("esp8266")){
+      System.err.println();
+      editor.statusError("SPIFFS Not Supported on "+PreferencesData.get("target_platform"));
+      return;
+    }
+    
+    if(!Base.getBoardPreferences().containsKey("build.spiffs_start") || !Base.getBoardPreferences().containsKey("build.spiffs_end")){
+      System.err.println();
+      editor.statusError("SPIFFS Not Defined for "+Base.getBoardPreferences().get("name"));
+      return;
+    }
+    long spiStart, spiEnd, spiPage, spiBlock;
+    try {
+      spiStart = getIntPref("build.spiffs_start");
+      spiEnd = getIntPref("build.spiffs_end");
+      spiPage = getIntPref("build.spiffs_pagesize");
+      if(spiPage == 0) spiPage = 256;
+      spiBlock = getIntPref("build.spiffs_blocksize");
+      if(spiBlock == 0) spiBlock = 4096;
+    } catch(Exception e){
+      editor.statusError(e);
+      return;
+    }
+    
+    TargetPlatform platform = Base.getTargetPlatform();
+    
+    File esptool;
+    if(!PreferencesData.get("runtime.os").contentEquals("windows")) esptool = new File(platform.getFolder()+"/tools", "esptool");
+    else esptool = new File(platform.getFolder()+"/tools", "esptool.exe");
+    if(!esptool.exists()){
+      System.err.println();
+      editor.statusError("SPIFFS Error: esptool not found!");
+      return;
+    }
+    
+    File tool;
+    if(!PreferencesData.get("runtime.os").contentEquals("windows")) tool = new File(platform.getFolder()+"/tools", "mkspiffs");
+    else tool = new File(platform.getFolder()+"/tools", "mkspiffs.exe");
+    if(!tool.exists()){
+      System.err.println();
+      editor.statusError("SPIFFS Error: mkspiffs not found!");
+      return;
+    }
+
+    int fileCount = 0;
+    File dataFolder = editor.getSketch().prepareDataFolder();
+    if(dataFolder.exists() && dataFolder.isDirectory()){
+      File[] files = dataFolder.listFiles();
+      if(files.length > 0){
+        for(File file : files){
+          if(!file.isDirectory() && file.isFile() && !file.getName().startsWith(".")) fileCount++;
+        }
+      }
+    }
+    
+    String dataPath = dataFolder.getAbsolutePath();
+    String toolPath = tool.getAbsolutePath();
+    String esptoolPath = esptool.getAbsolutePath();
+    String sketchName = editor.getSketch().getName();
+    String buildPath = Base.getBuildFolder().getAbsolutePath();
+    String imagePath = buildPath+"/"+sketchName+".spiffs.bin";
+    String serialPort = PreferencesData.get("serial.port");
+    String resetMethod = Base.getBoardPreferences().get("upload.resetmethod");
+    String uploadSpeed = Base.getBoardPreferences().get("upload.speed");
+    String uploadAddress = Base.getBoardPreferences().get("build.spiffs_start");
+    
+    Object[] options = { "Yes", "No" };
+    String title = "SPIFFS Create";
+    String message = "No files have been found in your data folder!\nAre you sure you want to create an empty SPIFFS image?";
+    
+    if(fileCount == 0 && JOptionPane.showOptionDialog(editor, message, title, JOptionPane.YES_NO_OPTION, JOptionPane.QUESTION_MESSAGE, null, options, options[1]) != JOptionPane.YES_OPTION){
+      System.err.println();
+      editor.statusError("SPIFFS Warning: mkspiffs canceled!");
+      return;
+    }
+    
+    editor.statusNotice("SPIFFS Creating Image...");
+    System.out.println("[SPIFFS] data   : "+dataPath);
+    System.out.println("[SPIFFS] size   : "+((spiEnd - spiStart)/1024));
+    System.out.println("[SPIFFS] page   : "+spiPage);
+    System.out.println("[SPIFFS] block  : "+spiBlock);
+    
+    try {
+      if(listenOnProcess(new String[]{toolPath, "-c", dataPath, "-p", spiPage+"", "-b", spiBlock+"", "-s", (spiEnd - spiStart)+"", imagePath}) != 0){
+        System.err.println();
+        editor.statusError("SPIFFS Create Failed!");
+        return;
+      }
+    } catch (Exception e){
+      editor.statusError(e);
+      editor.statusError("SPIFFS Create Failed!");
+      return;
+    }
+    
+    editor.statusNotice("SPIFFS Uploading Image...");
+    System.out.println("[SPIFFS] upload : "+imagePath);
+    System.out.println("[SPIFFS] reset  : "+resetMethod);
+    System.out.println("[SPIFFS] port   : "+serialPort);
+    System.out.println("[SPIFFS] speed  : "+uploadSpeed);
+    System.out.println("[SPIFFS] address: "+uploadAddress);
+    System.out.println();
+    
+    sysExec(new String[]{esptoolPath, "-cd", resetMethod, "-cb", uploadSpeed, "-cp", serialPort, "-ca", uploadAddress, "-cf", imagePath});
+  }
+  
+  public void run() {
+    createAndUpload();
+  }
+}

hardware/esp8266com/esp8266/boards.txt

@@ -25,6 +25,8 @@ generic.build.flash_freq=40
 generic.build.flash_ld=eagle.flash.512k.ld
 generic.build.spiffs_start=0x6B000
 generic.build.spiffs_end=0x7B000
+generic.build.spiffs_pagesize=256
+generic.build.spiffs_blocksize=4096
 
 generic.menu.CpuFrequency.80=80 MHz
 generic.menu.CpuFrequency.80.build.f_cpu=80000000L
@@ -55,46 +57,57 @@ generic.menu.FlashSize.512K.build.flash_size=512K
 generic.menu.FlashSize.512K.build.flash_ld=eagle.flash.512k.ld
 generic.menu.FlashSize.512K.build.spiffs_start=0x6B000
 generic.menu.FlashSize.512K.build.spiffs_end=0x7B000
+generic.menu.FlashSize.512K.build.spiffs_blocksize=4096
 generic.menu.FlashSize.1M512=1M (512K SPIFFS)
 generic.menu.FlashSize.1M512.build.flash_size=1M
 generic.menu.FlashSize.1M512.build.flash_ld=eagle.flash.1m512.ld
 generic.menu.FlashSize.1M512.build.spiffs_start=0x6B000
 generic.menu.FlashSize.1M512.build.spiffs_end=0xFB000
+generic.menu.FlashSize.1M512.build.spiffs_blocksize=8192
 generic.menu.FlashSize.1M256=1M (256K SPIFFS)
 generic.menu.FlashSize.1M256.build.flash_size=1M
 generic.menu.FlashSize.1M256.build.flash_ld=eagle.flash.1m256.ld
 generic.menu.FlashSize.1M256.build.spiffs_start=0xAB000
 generic.menu.FlashSize.1M256.build.spiffs_end=0xFB000
+generic.menu.FlashSize.1M256.build.spiffs_blocksize=4096
 generic.menu.FlashSize.1M128=1M (128K SPIFFS)
 generic.menu.FlashSize.1M128.build.flash_size=1M
 generic.menu.FlashSize.1M128.build.flash_ld=eagle.flash.1m128.ld
 generic.menu.FlashSize.1M128.build.spiffs_start=0xCB000
 generic.menu.FlashSize.1M128.build.spiffs_end=0xFB000
+generic.menu.FlashSize.1M128.build.spiffs_blocksize=4096
 generic.menu.FlashSize.1M64=1M (64K SPIFFS)
 generic.menu.FlashSize.1M64.build.flash_size=1M
 generic.menu.FlashSize.1M64.build.flash_ld=eagle.flash.1m64.ld
 generic.menu.FlashSize.1M64.build.spiffs_start=0xEB000
 generic.menu.FlashSize.1M64.build.spiffs_end=0xFB000
+generic.menu.FlashSize.1M64.build.spiffs_blocksize=4096
 generic.menu.FlashSize.2M=2M (1M SPIFFS)
 generic.menu.FlashSize.2M.build.flash_size=2M
 generic.menu.FlashSize.2M.build.flash_ld=eagle.flash.2m.ld
 generic.menu.FlashSize.2M.build.spiffs_start=0x100000
 generic.menu.FlashSize.2M.build.spiffs_end=0x1FB000
+generic.menu.FlashSize.2M.build.spiffs_blocksize=8192
 generic.menu.FlashSize.4M=4M (3M SPIFFS)
 generic.menu.FlashSize.4M.build.flash_size=4M
 generic.menu.FlashSize.4M.build.flash_ld=eagle.flash.4m.ld
 generic.menu.FlashSize.4M.build.spiffs_start=0x100000
 generic.menu.FlashSize.4M.build.spiffs_end=0x3FB000
-generic.menu.FlashSize.8M=8M (7M SPIFFS)
+generic.menu.FlashSize.4M.build.spiffs_blocksize=8192
-generic.menu.FlashSize.8M.build.flash_size=1M
+
-generic.menu.FlashSize.8M.build.flash_ld=eagle.flash.8m.ld
+# disabled because espressif's bootloader refuses to write above 4M
-generic.menu.FlashSize.8M.build.spiffs_start=0x100000
+# generic.menu.FlashSize.8M=8M (7M SPIFFS)
-generic.menu.FlashSize.8M.build.spiffs_end=0x800000
+# generic.menu.FlashSize.8M.build.flash_size=1M
-generic.menu.FlashSize.16M=16M (15M SPIFFS)
+# generic.menu.FlashSize.8M.build.flash_ld=eagle.flash.8m.ld
-generic.menu.FlashSize.16M.build.flash_size=1M
+# generic.menu.FlashSize.8M.build.spiffs_start=0x100000
-generic.menu.FlashSize.16M.build.flash_ld=eagle.flash.16m.ld
+# generic.menu.FlashSize.8M.build.spiffs_end=0x800000
-generic.menu.FlashSize.16M.build.spiffs_start=0x100000
+# generic.menu.FlashSize.8M.build.spiffs_blocksize=8192
-generic.menu.FlashSize.16M.build.spiffs_end=0x1000000
+# generic.menu.FlashSize.16M=16M (15M SPIFFS)
+# generic.menu.FlashSize.16M.build.flash_size=1M
+# generic.menu.FlashSize.16M.build.flash_ld=eagle.flash.16m.ld
+# generic.menu.FlashSize.16M.build.spiffs_start=0x100000
+# generic.menu.FlashSize.16M.build.spiffs_end=0x1000000
+# generic.menu.FlashSize.16M.build.spiffs_blocksize=8192
 
 # generic.menu.FlashFreq.40=40MHz
 # generic.menu.FlashFreq.40.build.flash_freq=40
@@ -124,6 +137,8 @@ modwifi.build.flash_freq=40
 modwifi.build.flash_ld=eagle.flash.2m.ld
 modwifi.build.spiffs_start=0x100000
 modwifi.build.spiffs_end=0x1FB000
+modwifi.build.spiffs_pagesize=256
+modwifi.build.spiffs_blocksize=8192
 
 modwifi.menu.CpuFrequency.80=80 MHz
 modwifi.menu.CpuFrequency.80.build.f_cpu=80000000L
@@ -172,6 +187,8 @@ nodemcu.build.flash_freq=40
 nodemcu.build.flash_ld=eagle.flash.4m.ld
 nodemcu.build.spiffs_start=0x100000
 nodemcu.build.spiffs_end=0x3FB000
+nodemcu.build.spiffs_pagesize=256
+nodemcu.build.spiffs_blocksize=8192
 
 nodemcu.menu.CpuFrequency.80=80 MHz
 nodemcu.menu.CpuFrequency.80.build.f_cpu=80000000L

hardware/esp8266com/esp8266/cores/esp8266/FileSystem.cpp

@@ -22,25 +22,25 @@
 #include "Arduino.h"
 #include "spiffs/spiffs_esp8266.h"
 
-#define LOGICAL_PAGE_SIZE       256
+// These addresses and sizes are defined in the linker script.
-#define LOGICAL_BLOCK_SIZE      (INTERNAL_FLASH_SECTOR_SIZE * 1)
-
-
-// These addresses are defined in the linker script.
 // For each flash memory size there is a linker script variant
 // which sets spiffs location and size.
 extern "C" uint32_t _SPIFFS_start;
 extern "C" uint32_t _SPIFFS_end;
+extern "C" uint32_t _SPIFFS_page;
+extern "C" uint32_t _SPIFFS_block;
 
 static s32_t api_spiffs_read(u32_t addr, u32_t size, u8_t *dst);
 static s32_t api_spiffs_write(u32_t addr, u32_t size, u8_t *src);
 static s32_t api_spiffs_erase(u32_t addr, u32_t size);
 
-FSClass FS((uint32_t) &_SPIFFS_start, (uint32_t) &_SPIFFS_end, 4);
+FSClass FS((uint32_t) &_SPIFFS_start, (uint32_t) &_SPIFFS_end, (uint32_t) &_SPIFFS_page, (uint32_t) &_SPIFFS_block, 4);
 
-FSClass::FSClass(uint32_t beginAddress, uint32_t endAddress, uint32_t maxOpenFiles)
+FSClass::FSClass(uint32_t beginAddress, uint32_t endAddress, uint32_t pageSize, uint32_t blockSize, uint32_t maxOpenFiles)
 : _beginAddress(beginAddress)
 , _endAddress(endAddress)
+, _pageSize(pageSize)
+, _blockSize(blockSize)
 , _maxOpenFiles(maxOpenFiles)
 , _fs({0})
 {
@@ -51,23 +51,25 @@ int FSClass::_mountInternal(){
     SPIFFS_API_DBG_E("Can't start file system, wrong address\r\n");
     return SPIFFS_ERR_NOT_CONFIGURED;
   }
+  if(_pageSize == 0) _pageSize = 256;
+  if(_blockSize == 0) _blockSize = 4096;
   
   spiffs_config cfg = {0};
   cfg.phys_addr = _beginAddress;
   cfg.phys_size = _endAddress - _beginAddress;
   cfg.phys_erase_block = INTERNAL_FLASH_SECTOR_SIZE;
-  cfg.log_block_size = LOGICAL_BLOCK_SIZE;
+  cfg.log_block_size = _blockSize;
-  cfg.log_page_size = LOGICAL_PAGE_SIZE;
+  cfg.log_page_size = _pageSize;
   cfg.hal_read_f = api_spiffs_read;
   cfg.hal_write_f = api_spiffs_write;
   cfg.hal_erase_f = api_spiffs_erase;
   
   SPIFFS_API_DBG_V("FSClass::_mountInternal: start:%x, size:%d Kb\n", cfg.phys_addr, cfg.phys_size / 1024);
 
-  _work.reset(new uint8_t[2*LOGICAL_PAGE_SIZE]);
+  _work.reset(new uint8_t[2*_pageSize]);
   _fdsSize = 32 * _maxOpenFiles;
   _fds.reset(new uint8_t[_fdsSize]);
-  _cacheSize = (32 + LOGICAL_PAGE_SIZE) * _maxOpenFiles;
+  _cacheSize = (32 + _pageSize) * _maxOpenFiles;
   _cache.reset(new uint8_t[_cacheSize]);
 
   s32_t res = SPIFFS_mount(&_fs,

hardware/esp8266com/esp8266/cores/esp8266/FileSystem.h

@@ -85,7 +85,7 @@ public:
 
 class FSClass {
 public:
-  FSClass(uint32_t beginAddress, uint32_t endAddress, uint32_t maxOpenFiles);
+  FSClass(uint32_t beginAddress, uint32_t endAddress, uint32_t pageSize, uint32_t blockSize, uint32_t maxOpenFiles);
 
   bool mount();
   void unmount();
@@ -117,6 +117,8 @@ protected:
   size_t _cacheSize;
   uint32_t _beginAddress;
   uint32_t _endAddress;
+  uint32_t _pageSize;
+  uint32_t _blockSize;
   uint32_t _maxOpenFiles;
   spiffs _fs;
 

hardware/esp8266com/esp8266/tools/sdk/ld/eagle.flash.16m.ld

@@ -13,5 +13,7 @@ MEMORY
 
 PROVIDE ( _SPIFFS_start = 0x40300000 );
 PROVIDE ( _SPIFFS_end = 0x41200000 );
+PROVIDE ( _SPIFFS_page = 0x100 );
+PROVIDE ( _SPIFFS_block = 0x2000 );
 
 INCLUDE "../ld/eagle.app.v6.common.ld"

hardware/esp8266com/esp8266/tools/sdk/ld/eagle.flash.1m128.ld

@@ -13,5 +13,7 @@ MEMORY
 
 PROVIDE ( _SPIFFS_start = 0x402DB000 );
 PROVIDE ( _SPIFFS_end = 0x402FB000 );
+PROVIDE ( _SPIFFS_page = 0x100 );
+PROVIDE ( _SPIFFS_block = 0x1000 );
 
 INCLUDE "../ld/eagle.app.v6.common.ld"

hardware/esp8266com/esp8266/tools/sdk/ld/eagle.flash.1m256.ld

@@ -13,5 +13,7 @@ MEMORY
 
 PROVIDE ( _SPIFFS_start = 0x402BB000 );
 PROVIDE ( _SPIFFS_end = 0x402FB000 );
+PROVIDE ( _SPIFFS_page = 0x100 );
+PROVIDE ( _SPIFFS_block = 0x1000 );
 
 INCLUDE "../ld/eagle.app.v6.common.ld"

hardware/esp8266com/esp8266/tools/sdk/ld/eagle.flash.1m512.ld

@@ -13,5 +13,7 @@ MEMORY
 
 PROVIDE ( _SPIFFS_start = 0x4027B000 );
 PROVIDE ( _SPIFFS_end = 0x402FB000 );
+PROVIDE ( _SPIFFS_page = 0x100 );
+PROVIDE ( _SPIFFS_block = 0x2000 );
 
 INCLUDE "../ld/eagle.app.v6.common.ld"

hardware/esp8266com/esp8266/tools/sdk/ld/eagle.flash.1m64.ld

@@ -13,5 +13,7 @@ MEMORY
 
 PROVIDE ( _SPIFFS_start = 0x402EB000 );
 PROVIDE ( _SPIFFS_end = 0x402FB000 );
+PROVIDE ( _SPIFFS_page = 0x100 );
+PROVIDE ( _SPIFFS_block = 0x1000 );
 
 INCLUDE "../ld/eagle.app.v6.common.ld"

hardware/esp8266com/esp8266/tools/sdk/ld/eagle.flash.2m.ld

@@ -13,5 +13,7 @@ MEMORY
 
 PROVIDE ( _SPIFFS_start = 0x40300000 );
 PROVIDE ( _SPIFFS_end = 0x403FB000 );
+PROVIDE ( _SPIFFS_page = 0x100 );
+PROVIDE ( _SPIFFS_block = 0x2000 );
 
 INCLUDE "../ld/eagle.app.v6.common.ld"

hardware/esp8266com/esp8266/tools/sdk/ld/eagle.flash.4m.ld

@@ -13,5 +13,7 @@ MEMORY
 
 PROVIDE ( _SPIFFS_start = 0x40300000 );
 PROVIDE ( _SPIFFS_end = 0x405FB000 );
+PROVIDE ( _SPIFFS_page = 0x100 );
+PROVIDE ( _SPIFFS_block = 0x2000 );
 
 INCLUDE "../ld/eagle.app.v6.common.ld"

hardware/esp8266com/esp8266/tools/sdk/ld/eagle.flash.512k.ld

@@ -13,5 +13,7 @@ MEMORY
 
 PROVIDE ( _SPIFFS_start = 0x4026B000 );
 PROVIDE ( _SPIFFS_end = 0x4027B000 );
+PROVIDE ( _SPIFFS_page = 0x100 );
+PROVIDE ( _SPIFFS_block = 0x1000 );
 
 INCLUDE "../ld/eagle.app.v6.common.ld"

hardware/esp8266com/esp8266/tools/sdk/ld/eagle.flash.8m.ld

@@ -13,5 +13,7 @@ MEMORY
 
 PROVIDE ( _SPIFFS_start = 0x40300000 );
 PROVIDE ( _SPIFFS_end = 0x40A00000 );
+PROVIDE ( _SPIFFS_page = 0x100 );
+PROVIDE ( _SPIFFS_block = 0x2000 );
 
 INCLUDE "../ld/eagle.app.v6.common.ld"