Allow GZIP compressed flash updates (#6820)

* Allow GZIP compressed flash updates

Modified the bootloader to be able to take stored updates in compressed
GZIP format (i.e. the output of "gzip -9 xxx.bin") and decompress them
on-the-fly to their final destination.  This can work for apps and for
filesystems (when used with the 2-step update option).

Allow eboot to be built using -Os/2 optimizations by fixing some portions
which failed when any optimizations were used.  Add -Wall and use data
and function sections to reduce size.  Use -Os to minimize size.

Remove obsolete esptool-ck calls to build a .ROM image, we don't use it.

Move all uninitted variables to RAM from IRAM, allowing 8-bit access.

Hook in @d-a-v and @pfalcon's uzlib port to actually do the
decompression.  Do not use any CRC checking which saves space.  Since we
have overwritten all of flash by the time we know id the CRC matches,
there's nothing we could have done anyway.

Adjust the Updater class to support GZIP files and not attempt to patch
them.

Bootloader builds to 0xd90 out of 0xfff bytes.

* Add @d-a-v's patch for httpupdate

https://github.com/esp8266/Arduino/pull/6820#pullrequestreview-326541014

* Update uzlib to point to pfalcon++

For now, because there are some self-test failures with @d-a-v's esp8266
branch (whose cool new features we don't actually use in eboot now)
start with pfalcon's 2.9 release and add the 2 patches (clcidx to code
from IRAM/RODATA, and the Windows test file renaming) needed to build
and run successfully.

* Add (c) notice for uzlib to README

.gitmodules

@@ -19,3 +19,6 @@
 [submodule "tools/esptool"]
 	path = tools/esptool
 	url = https://github.com/espressif/esptool.git
+[submodule "tools/sdk/uzlib"]
+	path = tools/sdk/uzlib
+	url = https://github.com/earlephilhower/uzlib.git

README.md

@@ -124,6 +124,8 @@ ESP8266 core files are licensed under LGPL.
 
 [LittleFS](https://github.com/ARMmbed/littlefs) library written by ARM Limited and released under the [BSD 3-clause license](https://github.com/ARMmbed/littlefs/blob/master/LICENSE.md).
 
+[uzlib](https://github.com/pfalcon/uzlib) library written and (c) 2014-2018 Paul Sokolovsky, licensed under the ZLib license (https://www.zlib.net/zlib_license.html).
+
 ### Other useful links ###
 
 [Toolchain repo](https://github.com/earlephilhower/esp-quick-toolchain)

bootloaders/eboot/Makefile

@@ -6,25 +6,30 @@ TARGET_DIR := ./
 
 TARGET_OBJ_FILES := \
 	eboot.o \
-	eboot_command.o \
+	eboot_command.o
-
 
 TARGET_OBJ_PATHS := $(addprefix $(TARGET_DIR)/,$(TARGET_OBJ_FILES))
 
+UZLIB_PATH := ../../tools/sdk/uzlib/src
+UZLIB_FLAGS := -DRUNTIME_BITS_TABLES
+
 CC := $(XTENSA_TOOLCHAIN)xtensa-lx106-elf-gcc
 CXX := $(XTENSA_TOOLCHAIN)xtensa-lx106-elf-g++
 AR := $(XTENSA_TOOLCHAIN)xtensa-lx106-elf-ar
 LD := $(XTENSA_TOOLCHAIN)xtensa-lx106-elf-gcc
 OBJDUMP := $(XTENSA_TOOLCHAIN)xtensa-lx106-elf-objdump
 
-INC += -I../../tools/sdk/include
+INC += -I../../tools/sdk/include -I../../tools/sdk/uzlib/src
+
 CFLAGS += -std=gnu99
 
-CFLAGS += -O0 -g -Wpointer-arith -Wno-implicit-function-declaration -Wl,-EL -fno-inline-functions -nostdlib -mlongcalls -mno-text-section-literals
+CFLAGS += -Os -g -Wall -Wpointer-arith -Wno-implicit-function-declaration -Wl,-EL -fno-inline-functions -nostdlib -mlongcalls -mno-text-section-literals -ffunction-sections -fdata-sections
 
 CFLAGS += $(INC)
 
-LDFLAGS	+= -nostdlib -Wl,--no-check-sections -umain
+CFLAGS += $(UZLIB_FLAGS)
+
+LDFLAGS	+= -nostdlib -Wl,--no-check-sections -Wl,--gc-sections -umain
 
 LD_SCRIPT := -Teboot.ld
 
@@ -32,19 +37,21 @@ APP_OUT:= eboot.elf
 APP_AR := eboot.a
 APP_FW := eboot.bin
 
-all: $(APP_FW)
 
-$(APP_AR): $(TARGET_OBJ_PATHS)
+all: $(APP_OUT)
+
+tinflate.o: $(UZLIB_PATH)/tinflate.c $(UZLIB_PATH)/uzlib.h $(UZLIB_PATH)/uzlib_conf.h
+	$(CC) $(CFLAGS) -c -o tinflate.o $(UZLIB_PATH)/tinflate.c
+
+tinfgzip.o: $(UZLIB_PATH)/tinfgzip.c $(UZLIB_PATH)/uzlib.h $(UZLIB_PATH)/uzlib_conf.h
+	$(CC) $(CFLAGS) -c -o tinfgzip.o $(UZLIB_PATH)/tinfgzip.c
+
+$(APP_AR): $(TARGET_OBJ_PATHS) tinflate.o tinfgzip.o
 	$(AR) cru $@ $^
 
-
 $(APP_OUT): $(APP_AR)
 	$(LD) $(LD_SCRIPT) $(LDFLAGS) -Wl,--start-group -Wl,--whole-archive $(APP_AR) -Wl,--end-group -o $@
 
-$(APP_FW): $(APP_OUT)
-	$(ESPTOOL) -vvv -eo $(APP_OUT) -bo $@ -bs .text -bs .data -bs .rodata -bc -ec || true
-
-
 clean:
 	rm -f *.o
 	rm -f $(APP_AR)

bootloaders/eboot/eboot.c

@@ -12,6 +12,9 @@
 #include <string.h>
 #include "flash.h"
 #include "eboot_command.h"
+#include <uzlib.h>
+
+extern unsigned char _gzip_dict;
 
 #define SWRST do { (*((volatile uint32_t*) 0x60000700)) |= 0x80000000; } while(0);
 
@@ -24,10 +27,14 @@ int print_version(const uint32_t flash_addr)
     if (SPIRead(flash_addr + APP_START_OFFSET + sizeof(image_header_t) + sizeof(section_header_t), &ver, sizeof(ver))) {
         return 1;
     }
-    const char* __attribute__ ((aligned (4))) fmtt = "v%08x\n\0\0";
+    char fmt[16];
-    uint32_t fmt[2];
+    fmt[0] = 'v';
-    fmt[0] = ((uint32_t*) fmtt)[0];
+    fmt[1] = '%';
-    fmt[1] = ((uint32_t*) fmtt)[1];
+    fmt[2] = '0';
+    fmt[3] = '8';
+    fmt[4] = 'x';
+    fmt[5] = '\n';
+    fmt[6] = '0';
     ets_printf((const char*) fmt, ver);
     return 0;
 }
@@ -80,38 +87,97 @@ int load_app_from_flash_raw(const uint32_t flash_addr)
         pos += section_header.size;
     }
 
-    register uint32_t sp asm("a1") = 0x3ffffff0;
+    asm volatile("" ::: "memory");
-    register uint32_t pc asm("a3") = image_header.entry;
+    asm volatile ("mov.n a1, %0\n"
-    __asm__  __volatile__ ("jx a3");
+        "mov.n a3, %1\n"
+        "jx a3\n" : : "r" (0x3ffffff0), "r" (image_header.entry) );
 
+    __builtin_unreachable(); // Save a few bytes by letting GCC know no need to pop regs/return
     return 0;
 }
 
+uint8_t read_flash_byte(const uint32_t addr)
+{
+    uint8_t __attribute__((aligned(4))) buff[4];
+    SPIRead(addr & ~3, buff, 4);
+    return buff[addr & 3];
+}
+unsigned char __attribute__((aligned(4))) uzlib_flash_read_cb_buff[4096];
+uint32_t uzlib_flash_read_cb_addr;
+int uzlib_flash_read_cb(struct uzlib_uncomp *m)
+{
+    m->source = uzlib_flash_read_cb_buff;
+    m->source_limit = uzlib_flash_read_cb_buff + sizeof(uzlib_flash_read_cb_buff);
+    SPIRead(uzlib_flash_read_cb_addr, uzlib_flash_read_cb_buff, sizeof(uzlib_flash_read_cb_buff));
+    uzlib_flash_read_cb_addr += sizeof(uzlib_flash_read_cb_buff);
+    return *(m->source++);
+}
 
+unsigned char gzip_dict[32768];
 
 int copy_raw(const uint32_t src_addr,
              const uint32_t dst_addr,
              const uint32_t size)
 {
     // require regions to be aligned
-    if (src_addr & 0xfff != 0 ||
+    if ((src_addr & 0xfff) != 0 ||
-        dst_addr & 0xfff != 0) {
+        (dst_addr & 0xfff) != 0) {
         return 1;
     }
 
     const uint32_t buffer_size = FLASH_SECTOR_SIZE;
     uint8_t buffer[buffer_size];
-    uint32_t left = ((size+buffer_size-1) & ~(buffer_size-1));
+    int32_t left = ((size+buffer_size-1) & ~(buffer_size-1));
     uint32_t saddr = src_addr;
     uint32_t daddr = dst_addr;
+    struct uzlib_uncomp m_uncomp;
+    bool gzip = false;
 
-    while (left) {
+    // Check if we are uncompressing a GZIP upload or not
+    if ((read_flash_byte(saddr) == 0x1f) && (read_flash_byte(saddr + 1) == 0x8b)) {
+        // GZIP signature matched.  Find real size as encoded at the end
+        left = read_flash_byte(saddr + size - 4);
+        left += read_flash_byte(saddr + size - 3)<<8;
+        left += read_flash_byte(saddr + size - 2)<<16;
+        left += read_flash_byte(saddr + size - 1)<<24;
+
+        uzlib_init();
+
+        /* all 3 fields below must be initialized by user */
+        m_uncomp.source = NULL;
+        m_uncomp.source_limit = NULL;
+        uzlib_flash_read_cb_addr = src_addr;
+        m_uncomp.source_read_cb = uzlib_flash_read_cb;
+        uzlib_uncompress_init(&m_uncomp, gzip_dict, sizeof(gzip_dict));
+
+        int res = uzlib_gzip_parse_header(&m_uncomp);
+        if (res != TINF_OK) {
+            return 5; // Error uncompress header read
+        }
+	gzip = true;
+    }
+    while (left > 0) {
         if (SPIEraseSector(daddr/buffer_size)) {
             return 2;
         }
+        if (!gzip) {
             if (SPIRead(saddr, buffer, buffer_size)) {
                 return 3;
             }
+        } else {
+            m_uncomp.dest_start = buffer;
+            m_uncomp.dest = buffer;
+            int to_read = (left > buffer_size) ? buffer_size : left;
+            m_uncomp.dest_limit = buffer + to_read;
+            int res = uzlib_uncompress(&m_uncomp);
+            if ((res != TINF_DONE) && (res != TINF_OK)) {
+                return 6;
+            }
+            // Fill any remaining with 0xff
+            for (int i = to_read; i < buffer_size; i++) {
+                buffer[i] = 0xff;
+            }
+        }
         if (SPIWrite(daddr, buffer, buffer_size)) {
             return 4;
         }
@@ -124,8 +190,7 @@ int copy_raw(const uint32_t src_addr,
 }
 
 
-
+int main()
-void main()
 {
     int res = 9;
     bool clear_cmd = false;
@@ -172,4 +237,7 @@ void main()
     }
 
     while(true){}
+
+    __builtin_unreachable();
+    return 0;
 }

bootloaders/eboot/eboot.ld

@@ -68,6 +68,8 @@ SECTIONS
 
   .data : ALIGN(4)
   {
+    *(COMMON)  /* Global vars */
+    . = ALIGN(4);
     _heap_start = ABSOLUTE(.);
 /*    _stack_sentry = ALIGN(0x8); */
   } >dram0_0_seg :dram0_0_bss_phdr
@@ -150,7 +152,6 @@ SECTIONS
     *(.bss)
     *(.bss.*)
     *(.gnu.linkonce.b.*)
-    *(COMMON)
     . = ALIGN (8);
     _bss_end = ABSOLUTE(.);
   } >iram1_0_seg :iram1_0_phdr

bootloaders/eboot/eboot_command.c

@@ -37,7 +37,7 @@ int eboot_command_read(struct eboot_command* cmd)
     }
 
     uint32_t crc32 = eboot_command_calculate_crc32(cmd);
-    if (cmd->magic & EBOOT_MAGIC_MASK != EBOOT_MAGIC || 
+    if ((cmd->magic & EBOOT_MAGIC_MASK) != EBOOT_MAGIC ||
         cmd->crc32 != crc32) {
         return 1;
     }

cores/esp8266/Updater.cpp

@@ -329,7 +329,8 @@ bool UpdaterClass::_writeBuffer(){
   bool modifyFlashMode = false;
   FlashMode_t flashMode = FM_QIO;
   FlashMode_t bufferFlashMode = FM_QIO;
-  if ((_currentAddress == _startAddress + FLASH_MODE_PAGE) && (_command == U_FLASH)) {
+  //TODO - GZIP can't do this
+  if ((_currentAddress == _startAddress + FLASH_MODE_PAGE) && (_buffer[0] != 0x1f) && (_command == U_FLASH)) {
     flashMode = ESP.getFlashChipMode();
     #ifdef DEBUG_UPDATER
       DEBUG_UPDATER.printf_P(PSTR("Header: 0x%1X %1X %1X %1X\n"), _buffer[0], _buffer[1], _buffer[2], _buffer[3]);
@@ -411,7 +412,7 @@ size_t UpdaterClass::write(uint8_t *data, size_t len) {
 bool UpdaterClass::_verifyHeader(uint8_t data) {
     if(_command == U_FLASH) {
         // check for valid first magic byte (is always 0xE9)
-        if(data != 0xE9) {
+        if ((data != 0xE9) && (data != 0x1f)) {
             _currentAddress = (_startAddress + _size);
             _setError(UPDATE_ERROR_MAGIC_BYTE);
             return false;
@@ -435,7 +436,12 @@ bool UpdaterClass::_verifyEnd() {
         }
 
         // check for valid first magic byte
-        if(buf[0] != 0xE9) {
+	//
+	// TODO: GZIP compresses the chipsize flags, so can't do check here
+	if ((buf[0] == 0x1f) && (buf[1] == 0x8b)) {
+            // GZIP, just assume OK
+            return true;
+        } else if (buf[0] != 0xE9) {
             _currentAddress = (_startAddress);
             _setError(UPDATE_ERROR_MAGIC_BYTE);            
             return false;

libraries/ESP8266httpUpdate/src/ESP8266httpUpdate.cpp

@@ -370,7 +370,7 @@ HTTPUpdateResult ESP8266HTTPUpdate::handleUpdate(HTTPClient& http, const String&
                     }
 
                     // check for valid first magic byte
-                    if(buf[0] != 0xE9) {
+                    if(buf[0] != 0xE9 && buf[0] != 0x1f) {
                         DEBUG_HTTP_UPDATE("[httpUpdate] Magic header does not start with 0xE9\n");
                         _setLastError(HTTP_UE_BIN_VERIFY_HEADER_FAILED);
                         http.end();
@@ -378,6 +378,7 @@ HTTPUpdateResult ESP8266HTTPUpdate::handleUpdate(HTTPClient& http, const String&
 
                     }
 
+                    if (buf[0] == 0xe9) {
                         uint32_t bin_flash_size = ESP.magicFlashChipSize((buf[3] & 0xf0) >> 4);
 
                         // check if new bin fits to SPI flash
@@ -388,6 +389,7 @@ HTTPUpdateResult ESP8266HTTPUpdate::handleUpdate(HTTPClient& http, const String&
                             return HTTP_UPDATE_FAILED;
                         }
                     }
+                }
                 if(runUpdate(*tcp, len, http.header("x-MD5"), command)) {
                     ret = HTTP_UPDATE_OK;
                     DEBUG_HTTP_UPDATE("[httpUpdate] Update ok\n");

tools/sdk/uzlib

@@ -0,0 +1 @@
+Subproject commit 80765a42d657fcd3a24e036ac9822f9dbf862449