arduino/esp8266
1
0
Fork 0
mirror of https://github.com/esp8266/Arduino.git synced 2025-04-19 23:22:16 +03:00
Code

Allow GZIP compressed flash updates (#6820)

Browse Source 
* Allow GZIP compressed flash updates

Modified the bootloader to be able to take stored updates in compressed
GZIP format (i.e. the output of "gzip -9 xxx.bin") and decompress them
on-the-fly to their final destination.  This can work for apps and for
filesystems (when used with the 2-step update option).

Allow eboot to be built using -Os/2 optimizations by fixing some portions
which failed when any optimizations were used.  Add -Wall and use data
and function sections to reduce size.  Use -Os to minimize size.

Remove obsolete esptool-ck calls to build a .ROM image, we don't use it.

Move all uninitted variables to RAM from IRAM, allowing 8-bit access.

Hook in @d-a-v and @pfalcon's uzlib port to actually do the
decompression.  Do not use any CRC checking which saves space.  Since we
have overwritten all of flash by the time we know id the CRC matches,
there's nothing we could have done anyway.

Adjust the Updater class to support GZIP files and not attempt to patch
them.

Bootloader builds to 0xd90 out of 0xfff bytes.

* Add @d-a-v's patch for httpupdate

https://github.com/esp8266/Arduino/pull/6820#pullrequestreview-326541014

* Update uzlib to point to pfalcon++

For now, because there are some self-test failures with @d-a-v's esp8266
branch (whose cool new features we don't actually use in eboot now)
start with pfalcon's 2.9 release and add the 2 patches (clcidx to code
from IRAM/RODATA, and the Windows test file renaming) needed to build
and run successfully.

* Add (c) notice for uzlib to README
This commit is contained in:
Earle F. Philhower, III 2019-12-18 09:17:38 -08:00 committed by GitHub
parent d40dbb4584
commit 1d0bc5efdf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 131 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitmodules vendored
View File

@ -19,3 +19,6 @@
[submodule "tools/esptool"]
path = tools/esptool
url = https://github.com/espressif/esptool.git
[submodule "tools/sdk/uzlib"]
path = tools/sdk/uzlib
url = https://github.com/earlephilhower/uzlib.git

2
README.md
View File

@ -124,6 +124,8 @@ ESP8266 core files are licensed under LGPL.
[LittleFS](https://github.com/ARMmbed/littlefs) library written by ARM Limited and released under the [BSD 3-clause license](https://github.com/ARMmbed/littlefs/blob/master/LICENSE.md).
[uzlib](https://github.com/pfalcon/uzlib) library written and (c) 2014-2018 Paul Sokolovsky, licensed under the ZLib license (https://www.zlib.net/zlib_license.html).
### Other useful links ###
[Toolchain repo](https://github.com/earlephilhower/esp-quick-toolchain)

31
bootloaders/eboot/Makefile
View File

@ -6,25 +6,30 @@ TARGET_DIR := ./
TARGET_OBJ_FILES := \
eboot.o \
eboot_command.o \
eboot_command.o
TARGET_OBJ_PATHS := $(addprefix $(TARGET_DIR)/,$(TARGET_OBJ_FILES))
UZLIB_PATH := ../../tools/sdk/uzlib/src
UZLIB_FLAGS := -DRUNTIME_BITS_TABLES
CC := $(XTENSA_TOOLCHAIN)xtensa-lx106-elf-gcc
CXX := $(XTENSA_TOOLCHAIN)xtensa-lx106-elf-g++
AR := $(XTENSA_TOOLCHAIN)xtensa-lx106-elf-ar
LD := $(XTENSA_TOOLCHAIN)xtensa-lx106-elf-gcc
OBJDUMP := $(XTENSA_TOOLCHAIN)xtensa-lx106-elf-objdump
INC += -I../../tools/sdk/include
INC += -I../../tools/sdk/include -I../../tools/sdk/uzlib/src
CFLAGS += -std=gnu99
CFLAGS += -O0 -g -Wpointer-arith -Wno-implicit-function-declaration -Wl,-EL -fno-inline-functions -nostdlib -mlongcalls -mno-text-section-literals
CFLAGS += -Os -g -Wall -Wpointer-arith -Wno-implicit-function-declaration -Wl,-EL -fno-inline-functions -nostdlib -mlongcalls -mno-text-section-literals -ffunction-sections -fdata-sections
CFLAGS += $(INC)
LDFLAGS += -nostdlib -Wl,--no-check-sections -umain
CFLAGS += $(UZLIB_FLAGS)
LDFLAGS += -nostdlib -Wl,--no-check-sections -Wl,--gc-sections -umain
LD_SCRIPT := -Teboot.ld
@ -32,19 +37,21 @@ APP_OUT:= eboot.elf
APP_AR := eboot.a
APP_FW := eboot.bin
all: $(APP_FW)
$(APP_AR): $(TARGET_OBJ_PATHS)
all: $(APP_OUT)
tinflate.o: $(UZLIB_PATH)/tinflate.c $(UZLIB_PATH)/uzlib.h $(UZLIB_PATH)/uzlib_conf.h
$(CC) $(CFLAGS) -c -o tinflate.o $(UZLIB_PATH)/tinflate.c
tinfgzip.o: $(UZLIB_PATH)/tinfgzip.c $(UZLIB_PATH)/uzlib.h $(UZLIB_PATH)/uzlib_conf.h
$(CC) $(CFLAGS) -c -o tinfgzip.o $(UZLIB_PATH)/tinfgzip.c
$(APP_AR): $(TARGET_OBJ_PATHS) tinflate.o tinfgzip.o
$(AR) cru $@ $^
$(APP_OUT): $(APP_AR)
$(LD) $(LD_SCRIPT) $(LDFLAGS) -Wl,--start-group -Wl,--whole-archive $(APP_AR) -Wl,--end-group -o $@
$(APP_FW): $(APP_OUT)
$(ESPTOOL) -vvv -eo $(APP_OUT) -bo $@ -bs .text -bs .data -bs .rodata -bc -ec || true
clean:
rm -f *.o
rm -f $(APP_AR)

100
bootloaders/eboot/eboot.c
View File

@ -12,6 +12,9 @@
#include <string.h>
#include "flash.h"
#include "eboot_command.h"
#include <uzlib.h>
extern unsigned char _gzip_dict;
#define SWRST do { (*((volatile uint32_t*) 0x60000700)) |= 0x80000000; } while(0);
@ -24,10 +27,14 @@ int print_version(const uint32_t flash_addr)
if (SPIRead(flash_addr + APP_START_OFFSET + sizeof(image_header_t) + sizeof(section_header_t), &ver, sizeof(ver))) {
return 1;
}
const char* __attribute__ ((aligned (4))) fmtt = "v%08x\n\0\0";
uint32_t fmt[2];
fmt[0] = ((uint32_t*) fmtt)[0];
fmt[1] = ((uint32_t*) fmtt)[1];
char fmt[16];
fmt[0] = 'v';
fmt[1] = '%';
fmt[2] = '0';
fmt[3] = '8';
fmt[4] = 'x';
fmt[5] = '\n';
fmt[6] = '0';
ets_printf((const char*) fmt, ver);
return 0;
}
@ -80,37 +87,96 @@ int load_app_from_flash_raw(const uint32_t flash_addr)
pos += section_header.size;
}
register uint32_t sp asm("a1") = 0x3ffffff0;
register uint32_t pc asm("a3") = image_header.entry;
__asm__ __volatile__ ("jx a3");
asm volatile("" ::: "memory");
asm volatile ("mov.n a1, %0\n"
"mov.n a3, %1\n"
"jx a3\n" : : "r" (0x3ffffff0), "r" (image_header.entry) );
__builtin_unreachable(); // Save a few bytes by letting GCC know no need to pop regs/return
return 0;
}
uint8_t read_flash_byte(const uint32_t addr)
{
uint8_t __attribute__((aligned(4))) buff[4];
SPIRead(addr & ~3, buff, 4);
return buff[addr & 3];
}
unsigned char __attribute__((aligned(4))) uzlib_flash_read_cb_buff[4096];
uint32_t uzlib_flash_read_cb_addr;
int uzlib_flash_read_cb(struct uzlib_uncomp *m)
{
m->source = uzlib_flash_read_cb_buff;
m->source_limit = uzlib_flash_read_cb_buff + sizeof(uzlib_flash_read_cb_buff);
SPIRead(uzlib_flash_read_cb_addr, uzlib_flash_read_cb_buff, sizeof(uzlib_flash_read_cb_buff));
uzlib_flash_read_cb_addr += sizeof(uzlib_flash_read_cb_buff);
return *(m->source++);
}
unsigned char gzip_dict[32768];
int copy_raw(const uint32_t src_addr,
const uint32_t dst_addr,
const uint32_t size)
{
// require regions to be aligned
if (src_addr & 0xfff != 0 ||
dst_addr & 0xfff != 0) {
if ((src_addr & 0xfff) != 0 ||
(dst_addr & 0xfff) != 0) {
return 1;
}
const uint32_t buffer_size = FLASH_SECTOR_SIZE;
uint8_t buffer[buffer_size];
uint32_t left = ((size+buffer_size-1) & ~(buffer_size-1));
int32_t left = ((size+buffer_size-1) & ~(buffer_size-1));
uint32_t saddr = src_addr;
uint32_t daddr = dst_addr;
struct uzlib_uncomp m_uncomp;
bool gzip = false;
while (left) {
// Check if we are uncompressing a GZIP upload or not
if ((read_flash_byte(saddr) == 0x1f) && (read_flash_byte(saddr + 1) == 0x8b)) {
// GZIP signature matched. Find real size as encoded at the end
left = read_flash_byte(saddr + size - 4);
left += read_flash_byte(saddr + size - 3)<<8;
left += read_flash_byte(saddr + size - 2)<<16;
left += read_flash_byte(saddr + size - 1)<<24;
uzlib_init();
/* all 3 fields below must be initialized by user */
m_uncomp.source = NULL;
m_uncomp.source_limit = NULL;
uzlib_flash_read_cb_addr = src_addr;
m_uncomp.source_read_cb = uzlib_flash_read_cb;
uzlib_uncompress_init(&m_uncomp, gzip_dict, sizeof(gzip_dict));
int res = uzlib_gzip_parse_header(&m_uncomp);
if (res != TINF_OK) {
return 5; // Error uncompress header read
}
gzip = true;
}
while (left > 0) {
if (SPIEraseSector(daddr/buffer_size)) {
return 2;
}
if (SPIRead(saddr, buffer, buffer_size)) {
return 3;
if (!gzip) {
if (SPIRead(saddr, buffer, buffer_size)) {
return 3;
}
} else {
m_uncomp.dest_start = buffer;
m_uncomp.dest = buffer;
int to_read = (left > buffer_size) ? buffer_size : left;
m_uncomp.dest_limit = buffer + to_read;
int res = uzlib_uncompress(&m_uncomp);
if ((res != TINF_DONE) && (res != TINF_OK)) {
return 6;
}
// Fill any remaining with 0xff
for (int i = to_read; i < buffer_size; i++) {
buffer[i] = 0xff;
}
}
if (SPIWrite(daddr, buffer, buffer_size)) {
return 4;
@ -124,13 +190,12 @@ int copy_raw(const uint32_t src_addr,
}
void main()
int main()
{
int res = 9;
bool clear_cmd = false;
struct eboot_command cmd;
print_version(0);
if (eboot_command_read(&cmd) == 0) {
@ -172,4 +237,7 @@ void main()
}
while(true){}
__builtin_unreachable();
return 0;
}

BIN
bootloaders/eboot/eboot.elf
View File

Binary file not shown.

3
bootloaders/eboot/eboot.ld
View File

@ -68,6 +68,8 @@ SECTIONS
.data : ALIGN(4)
{
*(COMMON) /* Global vars */
. = ALIGN(4);
_heap_start = ABSOLUTE(.);
/* _stack_sentry = ALIGN(0x8); */
} >dram0_0_seg :dram0_0_bss_phdr
@ -150,7 +152,6 @@ SECTIONS
*(.bss)
*(.bss.*)
*(.gnu.linkonce.b.*)
*(COMMON)
. = ALIGN (8);
_bss_end = ABSOLUTE(.);
} >iram1_0_seg :iram1_0_phdr

2
bootloaders/eboot/eboot_command.c
View File

@ -37,7 +37,7 @@ int eboot_command_read(struct eboot_command* cmd)
}
uint32_t crc32 = eboot_command_calculate_crc32(cmd);
if (cmd->magic & EBOOT_MAGIC_MASK != EBOOT_MAGIC ||
if ((cmd->magic & EBOOT_MAGIC_MASK) != EBOOT_MAGIC ||
cmd->crc32 != crc32) {
return 1;
}

12
cores/esp8266/Updater.cpp
View File

@ -329,7 +329,8 @@ bool UpdaterClass::_writeBuffer(){
bool modifyFlashMode = false;
FlashMode_t flashMode = FM_QIO;
FlashMode_t bufferFlashMode = FM_QIO;
if ((_currentAddress == _startAddress + FLASH_MODE_PAGE) && (_command == U_FLASH)) {
//TODO - GZIP can't do this
if ((_currentAddress == _startAddress + FLASH_MODE_PAGE) && (_buffer[0] != 0x1f) && (_command == U_FLASH)) {
flashMode = ESP.getFlashChipMode();
#ifdef DEBUG_UPDATER
DEBUG_UPDATER.printf_P(PSTR("Header: 0x%1X %1X %1X %1X\n"), _buffer[0], _buffer[1], _buffer[2], _buffer[3]);
@ -411,7 +412,7 @@ size_t UpdaterClass::write(uint8_t *data, size_t len) {
bool UpdaterClass::_verifyHeader(uint8_t data) {
if(_command == U_FLASH) {
// check for valid first magic byte (is always 0xE9)
if(data != 0xE9) {
if ((data != 0xE9) && (data != 0x1f)) {
_currentAddress = (_startAddress + _size);
_setError(UPDATE_ERROR_MAGIC_BYTE);
return false;
@ -435,7 +436,12 @@ bool UpdaterClass::_verifyEnd() {
}
// check for valid first magic byte
if(buf[0] != 0xE9) {
//
// TODO: GZIP compresses the chipsize flags, so can't do check here
if ((buf[0] == 0x1f) && (buf[1] == 0x8b)) {
// GZIP, just assume OK
return true;
} else if (buf[0] != 0xE9) {
_currentAddress = (_startAddress);
_setError(UPDATE_ERROR_MAGIC_BYTE);
return false;

18
libraries/ESP8266httpUpdate/src/ESP8266httpUpdate.cpp
View File

@ -370,7 +370,7 @@ HTTPUpdateResult ESP8266HTTPUpdate::handleUpdate(HTTPClient& http, const String&
}
// check for valid first magic byte
if(buf[0] != 0xE9) {
if(buf[0] != 0xE9 && buf[0] != 0x1f) {
DEBUG_HTTP_UPDATE("[httpUpdate] Magic header does not start with 0xE9\n");
_setLastError(HTTP_UE_BIN_VERIFY_HEADER_FAILED);
http.end();
@ -378,14 +378,16 @@ HTTPUpdateResult ESP8266HTTPUpdate::handleUpdate(HTTPClient& http, const String&
}
uint32_t bin_flash_size = ESP.magicFlashChipSize((buf[3] & 0xf0) >> 4);
if (buf[0] == 0xe9) {
uint32_t bin_flash_size = ESP.magicFlashChipSize((buf[3] & 0xf0) >> 4);
// check if new bin fits to SPI flash
if(bin_flash_size > ESP.getFlashChipRealSize()) {
DEBUG_HTTP_UPDATE("[httpUpdate] New binary does not fit SPI Flash size\n");
_setLastError(HTTP_UE_BIN_FOR_WRONG_FLASH);
http.end();
return HTTP_UPDATE_FAILED;
// check if new bin fits to SPI flash
if(bin_flash_size > ESP.getFlashChipRealSize()) {
DEBUG_HTTP_UPDATE("[httpUpdate] New binary does not fit SPI Flash size\n");
_setLastError(HTTP_UE_BIN_FOR_WRONG_FLASH);
http.end();
return HTTP_UPDATE_FAILED;
}
}
}
if(runUpdate(*tcp, len, http.header("x-MD5"), command)) {

1
tools/sdk/uzlib Submodule

@ -0,0 +1 @@
Subproject commit 80765a42d657fcd3a24e036ac9822f9dbf862449