Rework CI workflows (#8688)

- split workflows into separate files to trigger by path
  this should help out documentation and boards / eboot / pkg files
  updates, since those *wont* trigger usual build stuff anymore
- build*.sh whatever merged into just common.sh and build.sh
  trigger different parity builds, mod % rem and allow to set .ino list
  through the environment variable
- removes unnecessary temporary files, try to use more pipes
  move remaining ones into cache dir instead of PWD
- remove legacy TRAVIS env vars, use ESP8266_ARDUINO prefix for config
- remove Windows path workarounds
- hardware/ and ide/ directories are set through envionment
  do not force specific paths, simplify builds on local machine
- sketch list is set through environment. expicit paths for Windows and
  macOS builders. platformio also gets a real shuffled list instead of
  mod and rem magic numbers
- detect root of the repo through git cli, not base{name,dir} or relative paths

.github/workflows/build-host.yml

@@ -0,0 +1,45 @@
+# Run host test suite under valgrind for runtime checking of code.
+# Also, a quick test that the mocking builds work at all
+
+name: Build on host OS
+
+on:
+  pull_request:
+    paths-ignore:
+      - "boards.txt"
+      - "package/**"
+      - "tools/boards.txt.py"
+      - 'doc/**'
+
+permissions:
+  contents: read
+
+jobs:
+  host-tests:
+    name: Tests
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        submodules: true
+    - run: |
+        sudo apt update
+        sudo apt install valgrind lcov
+        bash ./tests/ci/host_test.sh
+
+  mock-check:
+    name: Mock
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        submodules: true
+    - run: |
+        cd tests/host
+        make -j ../../libraries/ESP8266WebServer/examples/FSBrowser/FSBrowser

.github/workflows/build-ide.yml

@@ -0,0 +1,96 @@
+# Cross-platform builds to ensure our Core and toolchain works
+
+name: Build IDE examples
+
+on:
+  pull_request:
+    paths-ignore:
+      - "boards.txt"
+      - "package/**"
+      - "tools/boards.txt.py"
+      - 'doc/**'
+
+permissions:
+  contents: read
+
+jobs:
+
+  # Examples are built in parallel to avoid CI total job time limitation
+
+  build-linux:
+    name: Linux - LwIP ${{ matrix.lwip }} (${{ matrix.chunk }})
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+    strategy:
+      matrix:
+        lwip: ["default", "IPv6"]
+        chunk: [0, 1, 2, 3, 4, 5, 6, 7]
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        submodules: true
+    - uses: actions/setup-python@v4
+      with:
+        python-version: '3.x'
+    - uses: actions/cache@v3
+      with:
+        path: ./tools/dist
+        key: ${{ runner.os }}-${{ hashFiles('package/package_esp8266com_index.template.json', 'tests/common.sh', 'tests/build.sh') }}
+    - name: Build Sketches
+      env:
+        ESP8266_ARDUINO_BUILDER: "arduino"
+        ESP8266_ARDUINO_IDE: "${{ runner.temp }}/arduino_ide"
+        ESP8266_ARDUINO_LWIP: ${{ matrix.lwip }}
+      run: |
+        bash ./tests/build.sh 8 ${{ matrix.chunk }}
+
+  # Just try to build at least one sketch, since we spend so much time with the matrix above
+  
+  build-windows:
+    name: Windows
+    runs-on: windows-latest
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        submodules: false
+    - uses: actions/setup-python@v4
+      with:
+        python-version: '3.x'
+    - uses: actions/cache@v3
+      with:
+        path: ./tools/dist
+        key: ${{ runner.os }}-${{ hashFiles('package/package_esp8266com_index.template.json', 'tests/common.sh', 'tests/build.sh') }}
+    - name: Build Sketch
+      env:
+        ESP8266_ARDUINO_HARDWARE: "${{ runner.temp }}/hardware"
+        ESP8266_ARDUINO_IDE: "${{ runner.temp }}/arduino_ide"
+        ESP8266_ARDUINO_SKETCHES: "libraries/esp8266/examples/Blink/Blink.ino"
+      run: |
+        bash ./tests/build.sh
+
+  build-mac:
+    name: macOS
+    runs-on: macOS-latest
+    defaults:
+      run:
+        shell: bash
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        submodules: false
+    - uses: actions/setup-python@v4
+      with:
+        python-version: '3.x'
+    - uses: actions/cache@v3
+      with:
+        path: ./tools/dist
+        key: ${{ runner.os }}-${{ hashFiles('package/package_esp8266com_index.template.json', 'tests/common.sh', 'tests/build.sh') }}
+    - name: Build Sketch
+      env:
+        ESP8266_ARDUINO_HARDWARE: "${{ runner.temp }}/hardware"
+        ESP8266_ARDUINO_IDE: "${{ runner.temp }}/arduino_ide"
+        ESP8266_ARDUINO_SKETCHES: "libraries/esp8266/examples/Blink/Blink.ino"
+      run: |
+        bash ./tests/build.sh

.github/workflows/build-platformio.yml

@@ -0,0 +1,43 @@
+# We do not distribute any environment settings, so just try to build some sketches
+# using the 'master' branch and using the uploaded toolchain version via get.py
+# Also, limit the amount of sketches and simply 
+
+name: Build examples with PlatformIO
+
+on:
+  pull_request:
+    paths-ignore:
+      - "boards.txt"
+      - "tools/boards.txt.py"
+      - 'doc/**'
+
+permissions:
+  contents: read
+
+jobs:
+  build-pio:
+    name: Linux (random sketches)
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        submodules: true
+    - uses: actions/setup-python@v4
+      with:
+        python-version: '3.x'
+    - uses: actions/cache@v3
+      with:
+        path: |
+          tools/dist
+          ~/.cache/pip
+          ~/.platformio/.cache
+        key: ${{ runner.os }}-${{ hashFiles('package/package_esp8266com_index.template.json', 'tests/common.sh', 'tests/build.sh') }}
+    - name: Build
+      env:
+        ESP8266_ARDUINO_BUILDER: "platformio"
+      run: |
+        pip install -U platformio
+        env ESP8266_ARDUINO_SKETCHES="$(find libraries/ -name '*.ino' | shuf -n 10 -)" bash ./tests/build.sh

.github/workflows/check-autogenerated.yml

@@ -0,0 +1,69 @@
+# Ensure no manual edits happen to our autogenerated files
+
+name: Check autogenerated files
+
+on:
+  pull_request:
+    paths:
+      - "boards.txt"
+      - "bootloaders/**"
+      - "doc/boards.rst"
+      - "package/**"
+      - "tools/boards.txt.py"
+      - "tools/sdk/ld/**"
+
+permissions:
+  contents: read
+
+jobs:
+  pkgrefs-check:
+    name: .json template
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        submodules: false
+    - run: |
+        bash ./tests/ci/pkgrefs_test.sh
+
+  eboot-check:
+    name: eboot .elf
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        submodules: false
+    - uses: actions/setup-python@v4
+      with:
+        python-version: '3.x'
+    - uses: actions/cache@v3
+      with:
+        path: ./tools/dist
+        key: ${{ runner.os }}-${{ hashFiles('package/package_esp8266com_index.template.json', 'tests/common.sh') }}
+    - run: |
+        # ^ reuse toolchain cache from our linux build job
+        git submodule update --init --remote tools/sdk/uzlib
+        bash ./tests/ci/eboot_test.sh
+
+  boards-txt-check:
+    name: boards.txt.py
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        submodules: false
+    - uses: actions/setup-python@v4
+      with:
+        python-version: '3.x'
+    - name: Check git-diff result
+      run: |
+        bash ./tests/ci/build_boards.sh

.github/workflows/documentation.yml

@@ -0,0 +1,30 @@
+# Ensure Sphinx can build the documentation properly.
+
+name: Documentation
+
+on:
+  pull_request:
+    paths:
+      - 'doc/**'
+
+permissions:
+  contents: read
+
+jobs:
+  documentation:
+    name: Sphinx build
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        submodules: true
+    - uses: actions/setup-python@v4
+      with:
+        python-version: '3.x'
+    - name: Build documentation
+      run: |
+        pip install --user -r doc/requirements.txt
+        bash ./tests/ci/build_docs.sh

.github/workflows/pull-request.yml

@@ -1,330 +0,0 @@
-# Run whenever a PR is generated or updated.
-
-# Most jobs check out the code, ensure Python3 is installed, and for build
-# tests the ESP8266 toolchain is cached when possible to speed up execution.
-
-name: ESP8266 Arduino CI
-
-on:
-  pull_request:
-
-
-permissions:
-  contents: read
-
-
-jobs:
-
-# Run 8 parallel jobs for the default build of all examples.
-  build-linux:
-    name: Build ${{ matrix.chunk }}
-    runs-on: ubuntu-latest
-    defaults:
-      run:
-        shell: bash
-    strategy:
-      matrix:
-        chunk: [0, 1, 2, 3, 4, 5, 6, 7]
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        submodules: true
-    - uses: actions/setup-python@v4
-      with:
-        python-version: '3.x'
-    - name: Cache Linux toolchain
-      id: cache-linux
-      uses: actions/cache@v3
-      with:
-        path: ./tools/dist
-        key: ${{ runner.os }}-${{ hashFiles('package/package_esp8266com_index.template.json', 'tests/common.sh') }}
-    - name: Build Sketches
-      env:
-        TRAVIS_BUILD_DIR: ${{ github.workspace }}
-        TRAVIS_TAG: ${{ github.ref }}
-        BUILD_PARITY: custom
-        mod: 8
-        rem: ${{ matrix.chunk }}
-      run: |
-        bash ./tests/build.sh
-
-
-# Cover the debug and IPv6 cases by enabling both and running 8 parallel jobs
-# over all example code.
-  build-debug-ipv6:
-    name: Debug IPv6 ${{ matrix.chunk }}
-    runs-on: ubuntu-latest
-    defaults:
-      run:
-        shell: bash
-    strategy:
-      matrix:
-        chunk: [0, 1, 2, 3, 4, 5, 6, 7]
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        submodules: true
-    - uses: actions/setup-python@v4
-      with:
-        python-version: '3.x'
-    - name: Cache Linux toolchain
-      id: cache-linux
-      uses: actions/cache@v3
-      with:
-        path: ./tools/dist
-        key: ${{ runner.os }}-${{ hashFiles('package/package_esp8266com_index.template.json', 'tests/common.sh') }}
-    - name: Build Sketches
-      env:
-        TRAVIS_BUILD_DIR: ${{ github.workspace }}
-        TRAVIS_TAG: ${{ github.ref }}
-        BUILD_PARITY: custom
-        mod: 8
-        rem: ${{ matrix.chunk }}
-      run: |
-        bash ./tests/debug6.sh
-
-
-# Single build under Windows to ensure the Win toolchain is good.
-  build-windows:
-    name: Windows
-    runs-on: windows-latest
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        submodules: true
-    - uses: actions/setup-python@v4
-      with:
-        python-version: '3.x'
-    - name: Cache Windows toolchain
-      id: cache-windows
-      uses: actions/cache@v3
-      with:
-        path: ./tools/dist
-        key: ${{ runner.os }}-${{ hashFiles('package/package_esp8266com_index.template.json', 'tests/common.sh') }}
-    - name: Build Sketch
-      env:
-        TRAVIS_BUILD_DIR: ${{ github.workspace }}
-        TRAVIS_TAG: ${{ github.ref }}
-        WINDOWS: 1
-        BUILD_PARITY: custom
-        mod: 500
-        rem: 1
-      run: |
-        # Windows has python3 already installed, but it's called "python".
-        # Copy python.exe to the proper name so scripts "just work".
-        try { Get-Command python3 } catch { copy (get-command python).source (get-command python).source.Replace("python.exe", "python3.exe") }
-        bash ./tests/build.sh
-
-
-# Single build under macOS to ensure the Mac toolchain is good.
-  build-mac:
-    name: Mac
-    runs-on: macOS-latest
-    defaults:
-      run:
-        shell: bash
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        submodules: true
-    - uses: actions/setup-python@v4
-      with:
-        python-version: '3.x'
-    - name: Cache Mac toolchain
-      id: cache-mac
-      uses: actions/cache@v3
-      with:
-        path: ./tools/dist
-        key: ${{ runner.os }}-${{ hashFiles('package/package_esp8266com_index.template.json', 'tests/common.sh') }}
-    - name: Build Sketch
-      env:
-        TRAVIS_BUILD_DIR: ${{ github.workspace }}
-        TRAVIS_TAG: ${{ github.ref }}
-        MACOSX: 1
-        BUILD_PARITY: custom
-        mod: 500
-        rem: 1
-      run: |
-        bash ./tests/build.sh
-
-
-# Run a few Platform.IO jobs (not full suite) to check PIO integration.
-  build-pio:
-    name: Build Platform.IO
-    runs-on: ubuntu-latest
-    defaults:
-      run:
-        shell: bash
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        submodules: true
-    - uses: actions/setup-python@v4
-      with:
-        python-version: '3.x'
-    - name: Build subset on Platform.IO
-      env:
-        TRAVIS_BUILD_DIR: ${{ github.workspace }}
-        TRAVIS_TAG: ${{ github.ref }}
-        BUILD_PARITY: custom
-        mod: 42 # Picked at random to give 4-5 builds and exit.
-        rem: 13
-      run: |
-        sudo apt update
-        sudo apt install python3-pip python3-setuptools
-        PATH=/home/runner/.local/bin:$PATH bash ./tests/platformio.sh
-
-
-# Run host test suite under valgrind for runtime checking of code.
-  host-tests:
-    name: Host tests
-    runs-on: ubuntu-latest
-    defaults:
-      run:
-        shell: bash
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        submodules: true
-    - uses: actions/setup-python@v4
-      with:
-        python-version: '3.x'
-    - name: Run host tests
-      env:
-        TRAVIS_BUILD_DIR: ${{ github.workspace }}
-        TRAVIS_TAG: ${{ github.ref }}
-      run: |
-        sudo apt update
-        sudo apt install valgrind lcov
-        bash ./tests/ci/host_test.sh
-
-
-# Ensure Sphinx can build the documentation properly.
-  documentation:
-    name: Documentation
-    runs-on: ubuntu-latest
-    defaults:
-      run:
-        shell: bash
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        submodules: true
-    - uses: actions/setup-python@v4
-      with:
-        python-version: '3.x'
-    - name: Build documentation
-      env:
-        TRAVIS_BUILD_DIR: ${{ github.workspace }}
-        TRAVIS_TAG: ${{ github.ref }}
-      run: |
-        sudo apt update
-        sudo apt install python3-pip python3-setuptools
-        # GitHub CI installs pip3 and setuptools outside the path.
-        # Update the path to include them and run.
-        PATH=/home/runner/.local/bin:$PATH pip3 install --user -r doc/requirements.txt
-        PATH=/home/runner/.local/bin:$PATH bash ./tests/ci/build_docs.sh
-
-
-# Standard Arduino formatting in all the examples
-  style-check:
-    name: Style and formatting
-    runs-on: ubuntu-latest
-    defaults:
-      run:
-        shell: bash
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        submodules: true
-    - uses: actions/setup-python@v4
-      with:
-        python-version: '3.x'
-    - name: Style check
-      env:
-        LLVM_SNAPSHOT_KEY: "6084F3CF814B57C1CF12EFD515CF4D18AF4F7421"
-        TRAVIS_BUILD_DIR: ${{ github.workspace }}
-        TRAVIS_TAG: ${{ github.ref }}
-      run: |
-          export GNUPGHOME=$(mktemp -d)
-          gpg --batch --keyserver keyserver.ubuntu.com --recv-keys "$LLVM_SNAPSHOT_KEY"
-          gpg --batch --armor --export "$LLVM_SNAPSHOT_KEY" | \
-            sudo tee /etc/apt/trusted.gpg.d/llvm-snapshot.gpg.asc
-          gpgconf --kill all
-          rm -r $GNUPGHOME
-          echo "deb http://apt.llvm.org/focal/     llvm-toolchain-focal-13   main" | \
-            sudo tee /etc/apt/sources.list.d/llvm.list
-          sudo apt update
-          sudo apt install clang-format-13
-          pip3 install pyyaml
-          bash ./tests/ci/style_check.sh
-
-
-# Quick test that the mocking builds succeed
-  mock-check:
-    name: Mock trivial test
-    runs-on: ubuntu-latest
-    defaults:
-      run:
-        shell: bash
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        submodules: true
-    - uses: actions/setup-python@v4
-      with:
-        python-version: '3.x'
-    - name: Mock build
-      env:
-        TRAVIS_BUILD_DIR: ${{ github.workspace }}
-        TRAVIS_TAG: ${{ github.ref }}
-      run: |
-          bash ./tests/buildm.sh
-
-
-# Ensure no manual edits to boards.txt
-  boards-check:
-    name: Boards.txt check
-    runs-on: ubuntu-latest
-    defaults:
-      run:
-        shell: bash
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        submodules: true
-    - uses: actions/setup-python@v4
-      with:
-        python-version: '3.x'
-    - name: Cache Linux toolchain
-      id: cache-linux
-      uses: actions/cache@v3
-      with:
-        path: ./tools/dist
-        key: ${{ runner.os }}-${{ hashFiles('package/package_esp8266com_index.template.json', 'tests/common.sh') }}
-    - name: Boards.txt diff
-      env:
-        TRAVIS_BUILD_DIR: ${{ github.workspace }}
-        TRAVIS_TAG: ${{ github.ref }}
-      run: |
-          bash ./tests/ci/build_boards.sh
-          bash ./tests/ci/eboot_test.sh
-          bash ./tests/ci/pkgrefs_test.sh
-
-
-# Validate orthography
-  code-spell:
-    name: Check spelling
-    runs-on: ubuntu-latest
-    defaults:
-      run:
-        shell: bash
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        submodules: true
-    - name: Run codespell
-      uses: codespell-project/actions-codespell@master
-      with:
-        skip: ./libraries/ESP8266SdFat,./libraries/LittleFS/lib,./tools/pyserial,./tools/sdk,./tools/esptool,./libraries/SoftwareSerial,./libraries/Ethernet,./github/workflows,./libraries/ESP8266HTTPUpdateServer/examples/SecureBearSSLUpdater/SecureBearSSLUpdater.ino,./libraries/esp8266/examples/RTCUserMemory/RTCUserMemory.ino,./libraries/esp8266/examples/StreamString/StreamString.ino,./libraries/ESP8266WiFi/examples/BearSSL_Validation/BearSSL_Validation.ino,./libraries/ESP8266WiFi/examples/BearSSL_Sessions/BearSSL_Sessions.ino,./libraries/ESP8266WebServer/examples/HelloServerBearSSL/HelloServerBearSSL.ino,./libraries/ESP8266WebServer/examples/HttpHashCredAuth/HttpHashCredAuth.ino,./cores/esp8266/spiffs,./tests/device/test_libc/libm_string.c, ./libraries/Netdump/examples/Netdump/Netdump.ino,./libraries/ESP8266WiFi/examples/BearSSL_Server,./cores/esp8266/LwipIntfDev.h
-        ignore_words_list: ESP8266,esp8266,esp,dout,DOUT,ser,ans

.github/workflows/release-to-publish.yml

@@ -35,9 +35,6 @@ jobs:
   package:
     name: Update master JSON file
     runs-on: ubuntu-latest
-    defaults:
-      run:
-        shell: bash
     steps:
     - uses: actions/checkout@v3
       with:
@@ -45,7 +42,6 @@ jobs:
         fetch-depth: 0
     - name: Deploy updated JSON
       env:
-        TRAVIS_BUILD_DIR: ${{ github.workspace }}
         BUILD_TYPE: package
         CI_GITHUB_API_KEY: ${{ secrets.GITHUB_TOKEN }}
         GHCI_DEPLOY_KEY: ${{ secrets.GHCI_DEPLOY_KEY }}

.github/workflows/style-check.yml

@@ -0,0 +1,56 @@
+name: Style and syntax checks
+
+on:
+  pull_request:
+
+permissions:
+  contents: read
+
+jobs:
+
+  # Generic formatting for Core and examples
+
+  clang-format:
+    name: clang-format
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        submodules: true
+    - uses: actions/setup-python@v4
+      with:
+        python-version: '3.x'
+    - name: Style check
+      env:
+        LLVM_SNAPSHOT_KEY: "6084F3CF814B57C1CF12EFD515CF4D18AF4F7421"
+      run: |
+          export GNUPGHOME=$(mktemp -d)
+          gpg --batch --keyserver keyserver.ubuntu.com --recv-keys "$LLVM_SNAPSHOT_KEY"
+          gpg --batch --armor --export "$LLVM_SNAPSHOT_KEY" | \
+            sudo tee /etc/apt/trusted.gpg.d/llvm-snapshot.gpg.asc
+          gpgconf --kill all
+          rm -r $GNUPGHOME
+          echo "deb http://apt.llvm.org/focal/     llvm-toolchain-focal-13   main" | \
+            sudo tee /etc/apt/sources.list.d/llvm.list
+          sudo apt update
+          sudo apt install clang-format-13
+          pip3 install pyyaml
+          bash ./tests/ci/style_check.sh
+
+  # Validate orthography
+
+  code-spell:
+    name: codespell
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        submodules: false
+    - name: Run codespell
+      uses: codespell-project/actions-codespell@master
+      with:
+        skip: ./libraries/ESP8266SdFat,./libraries/LittleFS/lib,./tools/pyserial,./tools/sdk,./tools/esptool,./libraries/SoftwareSerial,./libraries/Ethernet,./github/workflows,./libraries/ESP8266HTTPUpdateServer/examples/SecureBearSSLUpdater/SecureBearSSLUpdater.ino,./libraries/esp8266/examples/RTCUserMemory/RTCUserMemory.ino,./libraries/esp8266/examples/StreamString/StreamString.ino,./libraries/ESP8266WiFi/examples/BearSSL_Validation/BearSSL_Validation.ino,./libraries/ESP8266WiFi/examples/BearSSL_Sessions/BearSSL_Sessions.ino,./libraries/ESP8266WebServer/examples/HelloServerBearSSL/HelloServerBearSSL.ino,./libraries/ESP8266WebServer/examples/HttpHashCredAuth/HttpHashCredAuth.ino,./cores/esp8266/spiffs,./tests/device/test_libc/libm_string.c, ./libraries/Netdump/examples/Netdump/Netdump.ino,./libraries/ESP8266WiFi/examples/BearSSL_Server,./cores/esp8266/LwipIntfDev.h
+        ignore_words_list: ESP8266,esp8266,esp,dout,DOUT,ser,ans

.github/workflows/tag-to-draft-release.yml

@@ -28,18 +28,24 @@ jobs:
     - name: Set GIT tag name
       run: |
         # Sets an environment variable used in the next steps
-        TRAVIS_TAG="$(git describe --exact-match --tags)"
+        ESP8266_ARDUINO_RELEASE_TAG="$(git describe --exact-match --tags)"
-        echo "TRAVIS_TAG=${TRAVIS_TAG}" >> $GITHUB_ENV
+        echo "ESP8266_ARDUINO_RELEASE_TAG=${ESP8266_ARDUINO_RELEASE_TAG}" >> $GITHUB_ENV
     - name: Build package JSON
       env:
-        TRAVIS_BUILD_DIR: ${{ github.workspace }}
-        BUILD_TYPE: package
         CI_GITHUB_API_KEY: ${{ secrets.GITHUB_TOKEN }}
+        BUILD_TYPE: package
       run: |
         bash ./tests/ci/build_package.sh
-           pip3 install PyGithub
         # Create a draft release and upload the ZIP and JSON files.
         # This draft is not visible to normal users and needs to be
         # updated manually with release notes and published from the
         # GitHub web interface.
-           python3 ./package/upload_release.py --user "$GITHUB_ACTOR" --repo "$GITHUB_REPOSITORY" --token "$CI_GITHUB_API_KEY" --tag "$TRAVIS_TAG" --name "Release $TRAVIS_TAG" --msg "Update the draft with release notes before publishing." package/versions/*/*.zip package/versions/*/package_esp8266com_index.json
+        pip3 install PyGithub
+        python3 ./package/upload_release.py \
+          --user "$GITHUB_ACTOR" \
+          --repo "$GITHUB_REPOSITORY" \
+          --token "$CI_GITHUB_API_KEY" \
+          --tag "$ESP8266_ARDUINO_RELEASE_TAG" \
+          --name "Release ${ESP8266_ARDUINO_RELEASE_TAG}" \
+          --msg "Update the draft with release notes before publishing." \
+          package/versions/*/*.zip package/versions/*/package_esp8266com_index.json

libraries/DNSServer/examples/NAPTCaptivePortal/NAPTCaptivePortal.ino

@@ -80,6 +80,8 @@
 #include <ESP8266mDNS.h>
 #include <EEPROM.h>
 
+#include "WifiHttp.h"
+
 #define NAPT 1000
 #define NAPT_PORT 10
 

libraries/DNSServer/examples/NAPTCaptivePortal/WifiHttp.h

@@ -11,6 +11,10 @@
 // code there till it is correct then copy/paste back here. Then, you can move
 // comment boarders around until the C code is back in place.
 
+#pragma once
+
+#include <Arduino.h>
+
 #ifdef DEBUG_VIEW
 static const char configHead[] PROGMEM = R"EOF(
 <!--

package/build_boards_manager_package.sh

@@ -1,6 +1,5 @@
 #!/bin/bash
 
-
 if [ ! -z "${manualversion}" ]; then
 
     # manual-made release based on $manualversion

tests/build.sh

@@ -1,21 +1,85 @@
 #!/usr/bin/env bash
 
-cache_dir=$(mktemp -d)
+root=$(git rev-parse --show-toplevel)
 
-source "$TRAVIS_BUILD_DIR"/tests/common.sh
+ESP8266_ARDUINO_BUILD_DIR=${ESP8266_ARDUINO_BUILD_DIR:-$root}
+ESP8266_ARDUINO_BUILDER=${ESP8266_ARDUINO_BUILDER:-arduino}
+ESP8266_ARDUINO_PRESERVE_CACHE=${ESP8266_ARDUINO_PRESERVE_CACHE:-}
 
-if [ -z "$BUILD_PARITY" ]; then
+ESP8266_ARDUINO_IDE=${ESP8266_ARDUINO_IDE:-$HOME/arduino_ide}
-    mod=1
+ESP8266_ARDUINO_HARDWARE=${ESP8266_ARDUINO_HARDWARE:-$HOME/Arduino/hardware}
-    rem=0
+ESP8266_ARDUINO_LIBRARIES=${ESP8266_ARDUINO_LIBRARIES:-$HOME/Arduino/libraries}
-elif [ "$BUILD_PARITY" = "even" ]; then
+
+ESP8266_ARDUINO_DEBUG=${ESP8266_ARDUINO_DEBUG:-nodebug}
+ESP8266_ARDUINO_LWIP=${ESP8266_ARDUINO_LWIP:-default}
+ESP8266_ARDUINO_SKETCHES=${ESP8266_ARDUINO_SKETCHES:-}
+
+source "$root/tests/common.sh"
+
+cmd=${0##*/}
+usage="
+ENVIRONMENT:
+  ESP8266_ARDUINO_SKETCHES - list of .ino files; defaults to **all available examples**
+  ESP8266_ARDUINO_BUILDER - arduino or platformio
+
+  For Arduino IDE:
+    ESP8266_ARDUINO_IDE - path to the IDE (portable)
+    ESP8266_ARDUINO_HARDWARE - path to the hardware directory (usually, containing our repo)
+    ESP8266_ARDUINO_LIBRATIES - path to the libraries directory (external dependencies)
+    ESP8266_ARDUINO_DEBUG - debug or nodebug
+    ESP8266_ARDUINO_LWIP - v4 or v6
+
+USAGE:
+  $cmd <[even | odd]> - build every Nth, when '<N> % 2' is either even or odd
+  $cmd <mod> <rem> - build every Nth, when '<N> % <mod>' is equal to 'rem'
+  $cmd - build every .ino file from ESP8266_ARDUINO_SKETCHES
+"
+
+mod=1
+rem=0
+
+if [ "$#" -eq 1 ] ; then
+    case "$1" in
+    "-h")
+        echo "$usage"
+        exit 0
+        ;;
+    "even")
         mod=2
         rem=0
-elif [ "$BUILD_PARITY" = "odd" ]; then
+        ;;
+    "odd")
         mod=2
         rem=1
+        ;;
+    *)
+        echo 'Can either be even or odd'
+        exit 1
+        ;;
+    esac
+elif [ "$#" -eq 2 ] ; then
+    mod=$1
+    rem=$2
+elif [ "$#" -gt 2 ] ; then
+    echo "$usage"
+    exit 1
 fi
 
-install_arduino nodebug
+if [ -z "$ESP8266_ARDUINO_SKETCHES" ] ; then
-build_sketches_with_arduino "$mod" "$rem" lm2f
+    ESP8266_ARDUINO_SKETCHES=$(find $root/libraries -name *.ino | sort)
+fi
 
-rm -rf "$cache_dir"
+case "$ESP8266_ARDUINO_BUILDER" in
+"arduino")
+    install_arduino "$ESP8266_ARDUINO_DEBUG"
+    build_sketches_with_arduino "$mod" "$rem" "$ESP8266_ARDUINO_LWIP"
+    ;;
+"platformio")
+    install_platformio nodemcuv2
+    build_sketches_with_platformio "$mod" "$rem"
+    ;;
+*)
+    echo "Unknown builder! Must be either arduino or platformio"
+    exit 1
+    ;;
+esac

tests/build6.sh

@@ -1,22 +0,0 @@
-#!/usr/bin/env bash
-
-cache_dir=$(mktemp -d)
-
-source "$TRAVIS_BUILD_DIR"/tests/common.sh
-
-if [ -z "$BUILD_PARITY" ]; then
-    mod=1
-    rem=0
-elif [ "$BUILD_PARITY" = "even" ]; then
-    mod=2
-    rem=0
-elif [ "$BUILD_PARITY" = "odd" ]; then
-    mod=2
-    rem=1
-fi
-
-install_arduino nodebug
-build_sketches_with_arduino "$mod" "$rem" lm6f
-
-rm -rf "$cache_dir"
-

tests/buildm.sh

@@ -1,7 +0,0 @@
-#!/bin/sh
-
-set -e
-
-cd $(cd ${0%/*}; pwd)/host
-
-make -j ../../libraries/ESP8266WebServer/examples/FSBrowser/FSBrowser

tests/ci/build_boards.sh

@@ -4,11 +4,14 @@
 
 set -ev
 
-cd $TRAVIS_BUILD_DIR
+root=$(git rev-parse --show-toplevel)
 
+cd $root
 tools/boards.txt.py --boardsgen --ldgen --packagegen --docgen
 
-git diff --exit-code -- boards.txt \
+git diff --exit-code -- \
+    boards.txt \
     doc/boards.rst \
     tools/sdk/ld/
-git diff --exit-code -w -- package/package_esp8266com_index.template.json
+git diff --exit-code --ignore-all-space -- \
+    package/package_esp8266com_index.template.json

tests/ci/build_docs.sh

@@ -4,6 +4,5 @@
 
 set -ev
 
-cd $TRAVIS_BUILD_DIR/doc
+root=$(git rev-parse --show-toplevel)
-
+env SPHINXOPTS="-W" make -C $root/doc html
-SPHINXOPTS="-W" make html

tests/ci/build_package.sh

@@ -4,13 +4,16 @@
 
 set -ev
 
-export PKG_URL=https://github.com/esp8266/Arduino/releases/download/$TRAVIS_TAG/esp8266-$TRAVIS_TAG.zip
+root=$(git rev-parse --show-toplevel)
-export DOC_URL=https://arduino-esp8266.readthedocs.io/en/$TRAVIS_TAG/
+tag=$ESP8266_ARDUINO_RELEASE_TAG
+
+export PKG_URL=https://github.com/esp8266/Arduino/releases/download/$tag/esp8266-$tag.zip
+export DOC_URL=https://arduino-esp8266.readthedocs.io/en/$tag/
 
 if [ -z "$CI_GITHUB_API_KEY" ]; then
     echo "Github API key not set. Skip building the package."
     exit 0
 fi
 
-cd $TRAVIS_BUILD_DIR/package
+cd $root/package
 ./build_boards_manager_package.sh

tests/ci/eboot_test.sh

@@ -1,13 +1,14 @@
 #!/bin/bash
 
-READELF="$TRAVIS_BUILD_DIR/tools/xtensa-lx106-elf/bin/xtensa-lx106-elf-readelf"
-
 set -ev
 
-cd $TRAVIS_BUILD_DIR/tools
+root=$(git rev-parse --show-toplevel)
+READELF="$root/tools/xtensa-lx106-elf/bin/xtensa-lx106-elf-readelf"
+
+cd $root/tools
 python3 get.py -q
 
-cd $TRAVIS_BUILD_DIR/bootloaders/eboot
+cd $root/bootloaders/eboot
 
 "$READELF" -x .data -x .text eboot.elf > git.txt
 make clean

tests/ci/host_test.sh

@@ -4,12 +4,8 @@
 
 set -ev
 
-if [ -z "$TRAVIS_BUILD_DIR" ]; then
+root=$(git rev-parse --show-toplevel)
-    export TRAVIS_BUILD_DIR=$(cd ${PWD%/*}/..; pwd)
+cd $root/tests/host
-fi
-
-cd $TRAVIS_BUILD_DIR/tests/host
-
 
 make -j2 FORCE32=0 ssl
 for i in ../../libraries/ESP8266WiFi/examples/WiFiClient/WiFiClient \

tests/ci/pkgrefs_test.sh

@@ -2,8 +2,10 @@
 
 set -ev
 
+root=$(git rev-parse --show-toplevel)
+
 fail=0
-for i in $(cat "$TRAVIS_BUILD_DIR/package/package_esp8266com_index.template.json" | jq '.packages[0]."tools" | .[] | .systems[] | "\(.url) \(.checksum)"' | sort -u | sed 's/ /@/'); do
+for i in $(cat "$root/package/package_esp8266com_index.template.json" | jq '.packages[0]."tools" | .[] | .systems[] | "\(.url) \(.checksum)"' | sort -u | sed 's/ /@/'); do
     url=$(echo $i | sed 's/@/ /' | cut -f2 -d\" | cut -f1 -d' ')
     sha=$(echo $i | sed 's/@/ /' | cut -f2 -d\" | cut -f2 -d' ' | cut -f2 -d:)
     echo "INFO:  Checking $url"

tests/ci/style_check.sh

@@ -4,12 +4,15 @@
 
 set -e -x
 
-root=$(git rev-parse --show-toplevel)
-${root}/tests/restyle.sh
-
-# Revert changes which formatter might have done to the submodules,
-# as we don't want to fail the build because of the 3rd party libraries
 git --version || true
-git submodule foreach --recursive 'git reset --hard'
+root=$(git rev-parse --show-toplevel)
 
-git diff --exit-code -- $TRAVIS_BUILD_DIR
+# Run formatter and compare what changed in the git tree.
+# Also revert changes which formatter might have done to the submodules,
+# as we don't want to fail the build because of the 3rd party libraries
+
+cd $root
+./tests/restyle.sh
+
+git submodule foreach --recursive 'git reset --hard'
+git diff --exit-code

tests/common.sh

@@ -1,78 +1,145 @@
 #!/usr/bin/env bash
 
-# return 1 if this test should not be built in CI (for other archs, not needed, etc.)
+set -u -e -E -o pipefail
+
+cache_dir=$(mktemp -d)
+trap 'trap_exit' EXIT
+
+function trap_exit()
+{
+    # workaround for macOS shipping with broken bash
+    local exit_code=$?
+    if [ -z "${ESP8266_ARDUINO_PRESERVE_CACHE-}" ]; then
+        rm -rf "$cache_dir"
+    fi
+
+    exit $exit_code
+}
+
+function step_summary()
+{
+    local header=$1
+    local contents=$2
+
+    # ref. https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#adding-a-job-summary
+    if [ -n "${GITHUB_STEP_SUMMARY-}" ]; then
+        { echo "# $header"; echo '```console'; cat "$contents"; echo '```'; } \
+            >> $GITHUB_STEP_SUMMARY
+    else
+        echo "# $header"
+        cat "$contents"
+    fi
+}
+
+# return 0 if this sketch should not be built in CI (for other archs, not needed, etc.)
 function skip_ino()
 {
-    local ino=$1
+    case $1 in
-    local skiplist=""
+    *"/#attic/"* | \
-    # Add items to the following list with "\n" netween them to skip running.  No spaces, tabs, etc. allowed
+    *"/AvrAdcLogger/"* | \
-    read -d '' skiplist << EOL || true
+    *"/examplesV1/"* | \
-/#attic/
+    *"/RtcTimestampTest/"* | \
-/AvrAdcLogger/
+    *"/SoftwareSpi/"* | \
-/examplesV1/
+    *"/TeensyDmaAdcLogger/"* | \
-/RtcTimestampTest/
+    *"/TeensyRtcTimestamp/"* | \
-/SoftwareSpi/
+    *"/TeensySdioDemo/"* | \
-/TeensyDmaAdcLogger/
+    *"/TeensySdioLogger/"* | \
-/TeensyRtcTimestamp/
+    *"/UserChipSelectFunction/"* | \
-/TeensySdioDemo/
+    *"/UserSPIDriver/"*)
-/TeensySdioLogger/
+        return 0
-/UserChipSelectFunction/
+        ;;
-/UserSPIDriver/
+    *"Teensy"*)
-EOL
+        return 0
-    echo $ino | grep -q -F "$skiplist"
+        ;;
-    echo $(( 1 - $? ))
+    *)
+        ;;
+    esac
+
+    return 1
+}
+
+# return reason if this sketch is not the main one or it is explicitly disabled with .test.skip in its directory
+function skip_sketch()
+{
+    local sketch=$1
+    local sketchname=$2
+    local sketchdir=$3
+    local sketchdirname=$4
+
+    if [[ "${sketchdirname}.ino" != "$sketchname" ]]; then
+        echo "Skipping $sketch (not the main sketch file)"
+    fi
+    if skip_ino "$sketch" || [[ -f "$sketchdir/.test.skip" ]]; then
+        echo "Skipping $sketch"
+    fi
+}
+
+function print_size_info_header()
+{
+    printf "%-28s %-8s %-8s %-8s %-8s %-10s %-8s %-8s\n" sketch data rodata bss text irom0.text dram flash
 }
 
 function print_size_info()
 {
-    elf_file=$1
+    local awk_script='
+/^\.data/ || /^\.rodata/ || /^\.bss/ || /^\.text/ || /^\.irom0\.text/{
+    size[$1] = $2
+}
 
-    if [ -z "$elf_file" ]; then
+END {
-        printf "sketch                       data     rodata   bss      text     irom0.text   dram     flash\n"
+    total_ram = size[".data"] + size[".rodata"] + size[".bss"]
-        return 0
+    total_flash = size[".data"] + size[".rodata"] + size[".text"] + size[".irom0.text"]
-    fi
 
+    printf "%-28s %-8d %-8d %-8d %-8d %-10d %-8d %-8d\n",
+            sketch_name,
+            size[".data"], size[".rodata"], size[".bss"], size[".text"], size[".irom0.text"],
+            total_ram, total_flash
+}
+'
+    local size=$1
+    local elf_file=$2
+
+    local elf_name
     elf_name=$(basename $elf_file)
-    sketch_name="${elf_name%.*}"
+    $size --format=sysv "$elf_file" | \
-    # echo $sketch_name
+        awk -v sketch_name="${elf_name%.*}" "$awk_script" -
-    xtensa-lx106-elf-size --format=sysv $elf_file | sed s/irom0.text/irom0text/g > size.txt
-    declare -A segments
-    for seg in data rodata bss text irom0text; do
-        segments[$seg]=$(grep ^.$seg size.txt | awk '{sum += $2} END {print sum}')
-    done
-
-    total_ram=$((${segments[data]} + ${segments[rodata]} + ${segments[bss]}))
-    total_flash=$((${segments[data]} + ${segments[rodata]} + ${segments[text]} + ${segments[irom0text]}))
-
-    printf "%-28s %-8d %-8d %-8d %-8d %-8d     %-8d %-8d\n" $sketch_name ${segments[data]} ${segments[rodata]} ${segments[bss]} ${segments[text]} ${segments[irom0text]} $total_ram $total_flash
-    return 0
 }
 
 function build_sketches()
 {
-    set +e
+    local core_path=$1
-    local arduino=$1
+    local ide_path=$2
-    local srcpath=$2
+    local hardware_path=$3
-    local build_arg=$3
+    local library_path=$4
-    local build_dir=build.tmp
+    local build_mod=$5
-    local build_mod=$4
+    local build_rem=$6
-    local build_rem=$5
+    local lwip=$7
-    local lwip=$6
+
-    mkdir -p $build_dir
+    local build_dir="$cache_dir"/build
-    local build_cmd="python3 tools/build.py -b generic -v -w all -s 4M1M -v -k --build_cache $cache_dir -p ./$build_dir -n $lwip $build_arg "
+    mkdir -p "$build_dir"
-    if [ "$WINDOWS" = "1" ]; then
+
-        # Paths to the arduino builder need to be / referenced, not our native ones
+    local build_cache="$cache_dir"/cache
-        build_cmd=$(echo $build_cmd --ide_path $arduino | sed 's/ \/c\// \//g' ) # replace '/c/' with '/'
+    mkdir -p "$build_cache"
-    fi
+
-    local sketches=$(find $srcpath -name *.ino | sort)
+    local build_cmd
-    print_size_info >size.log
+    build_cmd="python3 tools/build.py"\
-    export ARDUINO_IDE_PATH=$arduino
+" --build_cache $build_cache"\
-    local k_partial_core_cleanup=("build.opt" "*.ino.globals.h")
+" --build_path $build_dir"\
+" --hardware_path $hardware_path"\
+" --ide_path $ide_path"\
+" --library_path $library_path"\
+" --lwIP $lwip"\
+" --board_name generic --verbose --warnings all"\
+" --flash_size 4M1M --keep"
+
+    print_size_info_header >"$cache_dir"/size.log
+
     local mk_clean_core=1
     local testcnt=0
-    for sketch in $sketches; do
+
+    for sketch in $ESP8266_ARDUINO_SKETCHES; do
         testcnt=$(( ($testcnt + 1) % $build_mod ))
-        if [ $testcnt -ne $build_rem ]; then
+        if [ $testcnt -ne "$build_rem" ]; then
             continue  # Not ours to do
         fi
 
@@ -90,20 +157,18 @@ function build_sketches()
             mk_clean_core=1
         fi
         if [ $mk_clean_core -ne 0 ]; then
-            rm -rf rm $build_dir/core/*
+            rm -rf "$build_dir"/core/*
         else
             # Remove sketch specific files from ./core/ between builds.
-            rm -rf $build_dir/core/build.opt $build_dir/core/*.ino.globals.h
+            rm -rf "$build_dir/core/build.opt" "$build_dir"/core/*.ino.globals.h
         fi
 
         if [ -e $cache_dir/core/*.a ]; then
             # We need to preserve the build.options.json file and replace the last .ino
             # with this sketch's ino file, or builder will throw everything away.
-            jq '."sketchLocation" = "'$sketch'"' $build_dir/build.options.json > $build_dir/build.options.json.tmp
+            jq '."sketchLocation" = "'$sketch'"' $build_dir/build.options.json \
-            mv $build_dir/build.options.json.tmp $build_dir/build.options.json
+                > "$build_dir"/build.options.json.tmp
-            # Set the time of the cached core.a file to the future so the GIT header
+            mv "$build_dir"/build.options.json.tmp "$build_dir"/build.options.json
-            # we regen won't cause the builder to throw it out and rebuild from scratch.
-            touch -d 'now + 1 day' $cache_dir/core/*.a
             if [ $mk_clean_core -ne 0 ]; then
                 # Hack workaround for CI not handling core rebuild for global options
                 rm $cache_dir/core/*.a
@@ -120,167 +185,332 @@ function build_sketches()
         # Clear out the last built sketch, map, elf, bin files, but leave the compiled
         # objects in the core and libraries available for use so we don't need to rebuild
         # them each sketch.
-        rm -rf $build_dir/sketch $build_dir/*.bin $build_dir/*.map $build_dir/*.elf
+        rm -rf "$build_dir"/sketch \
+            "$build_dir"/*.bin \
+            "$build_dir"/*.map \
+            "$build_dir"/*.elf
 
-        local sketchdir=$(dirname $sketch)
+        local sketchdir
-        local sketchdirname=$(basename $sketchdir)
+        sketchdir=$(dirname $sketch)
-        local sketchname=$(basename $sketch)
+
-        if [[ "${sketchdirname}.ino" != "$sketchname" ]]; then
+        local sketchdirname
-            echo "Skipping $sketch, because it is not the main sketch file";
+        sketchdirname=$(basename $sketchdir)
-            continue
+
-        fi;
+        local sketchname
-        if [[ -f "$sketchdir/.test.skip" ]]; then
+        sketchname=$(basename $sketch)
-            echo -e "\n ------------ Skipping $sketch ------------ \n";
+
+        local skip
+        skip=$(skip_sketch "$sketch" "$sketchname" "$sketchdir" "$sketchdirname")
+        if [ -n "$skip" ]; then
+            echo "$skip"
             continue
         fi
-        if [[ $(skip_ino $sketch) = 1 ]]; then
+
-            echo -e "\n ------------ Skipping $sketch ------------ \n";
+        echo ::group::Building $sketch
-            continue
-        fi
-        echo -e "\n ------------ Building $sketch ------------ \n";
-        # $arduino --verify $sketch;
-        if [ "$WINDOWS" == "1" ]; then
-            sketch=$(echo $sketch | sed 's/^\/c//')
-            # MINGW will try to be helpful and silently convert args that look like paths to point to a spot inside the MinGW dir.  This breaks everything.
-            # http://www.mingw.org/wiki/Posix_path_conversion
-            # https://stackoverflow.com/questions/7250130/how-to-stop-mingw-and-msys-from-mangling-path-names-given-at-the-command-line#34386471
-            export MSYS2_ARG_CONV_EXC="*"
-            export MSYS_NO_PATHCONV=1
-        fi
         echo "$build_cmd $sketch"
-        time ($build_cmd $sketch >build.log)
+
-        local result=$?
+        local result
+        time $build_cmd $sketch >"$cache_dir"/build.log \
+            && result=0 || result=1
+
         if [ $result -ne 0 ]; then
-            echo "Build failed ($1)"
+            echo ::error::Build failed for $sketch
-            echo "Build log:"
+            cat "$cache_dir/build.log"
-            cat build.log
+            echo ::endgroup::
-            set -e
             return $result
         else
-            local warns=$( grep -c warning: build.log )
+            grep -s -c warning: "$cache_dir"/build.log \
-            if [ $warns -ne 0 ]; then
+                && step_summary "$sketch warnings" "$cache_dir/build.log"
-                echo "Warnings detected, log follows:"
-                cat build.log
         fi
-        fi
+
-        rm build.log
+        print_size_info "$core_path"/tools/xtensa-lx106-elf/bin/xtensa-lx106-elf-size \
-        print_size_info $build_dir/*.elf >>size.log
+            $build_dir/*.elf >>$cache_dir/size.log
+
+        echo ::endgroup::
     done
-    set -e
+}
+
+function check_hash()
+{
+    local file=$1
+    local hash=$2
+
+    local shasum
+    case ${RUNNER_OS-} in
+    "macOS")
+        shasum="shasum -a 512"
+        ;;
+    *)
+        shasum="sha512sum"
+        ;;
+    esac
+
+    echo "$hash  $file" | $shasum -c -
+}
+
+function fetch_and_unpack()
+{
+    local archive=$1
+    local hash=$2
+    local url=$3
+
+    test -r "$archive" \
+        && check_hash "$archive" "$hash" \
+        || { pushd "$cache_dir"
+             curl --output "$archive" --location "$url";
+             check_hash "$archive" "$hash";
+             popd;
+             mv "$cache_dir/$archive" ./"$archive"; }
+
+    case $archive in
+    *".zip")
+        unzip -q "$archive"
+        ;;
+    *)
+        tar xf "$archive"
+        ;;
+    esac
+}
+
+function install_library()
+{
+    local lib_path=$1
+    local name=$2
+    local archive=$3
+    local hash=$4
+    local url=$5
+
+    fetch_and_unpack "$archive" "$hash" "$url"
+    mkdir -p "$lib_path"
+    rm -rf "$lib_path/$name"
+    mv "$name" "$lib_path/$name"
 }
 
 function install_libraries()
 {
-    mkdir -p $HOME/Arduino/libraries
+    local core_path=$1
-    pushd $HOME/Arduino/libraries
+    local lib_path=$2
 
-    # install ArduinoJson library
+    mkdir -p "$core_path"/tools/dist
-    { test -r ArduinoJson-v6.11.0.zip || curl --output ArduinoJson-v6.11.0.zip -L https://github.com/bblanchon/ArduinoJson/releases/download/v6.11.0/ArduinoJson-v6.11.0.zip; } && unzip -q ArduinoJson-v6.11.0.zip
+    pushd "$core_path"/tools/dist
+
+    install_library "$lib_path" \
+        "ArduinoJson" \
+        "ArduinoJson-v6.11.5.zip" \
+        "8b836c862e69e60c4357a5ed7cbcf1310a3bb1c6bd284fe028faaa3d9d7eed319d10febc8a6a3e06040d1c73aaba5ca487aeffe87ae9388dc4ae1677a64d602c" \
+        "https://github.com/bblanchon/ArduinoJson/releases/download/v6.11.5/ArduinoJson-v6.11.5.zip"
 
     popd
 }
 
 function install_ide()
 {
-    local idever='nightly'
+    # TODO replace ide distribution + arduino-builder with arduino-cli
-    #local idever='1.8.10'
+    local idever='1.8.19'
-
-    #local ideurl='https://www.arduino.cc/download.php?f=/arduino-nightly'
     local ideurl="https://downloads.arduino.cc/arduino-$idever"
 
-    echo "using Arduino IDE distribution ${idever}"
+    echo "Arduino IDE ${idever}"
+
+    local core_path=$1
+    local ide_path=$2
 
-    local ide_path=$1
-    local core_path=$2
-    local debug=$3
     mkdir -p ${core_path}/tools/dist
-    if [ "$WINDOWS" = "1" ]; then
+    pushd ${core_path}/tools/dist
-        test -r ${core_path}/tools/dist/arduino-windows.zip || curl --output ${core_path}/tools/dist/arduino-windows.zip -L "${ideurl}-windows.zip"
+
-        unzip -q ${core_path}/tools/dist/arduino-windows.zip
+    if [ "${RUNNER_OS-}" = "Windows" ]; then
-        mv arduino-${idever} arduino-distrib
+        fetch_and_unpack "arduino-windows.zip" \
-    elif [ "$MACOSX" = "1" ]; then
+            "c4072d808aea3848bceff5772f9d1e56a0fde02366b5aa523d10975c54eee2ca8def25ee466abbc88995aa323d475065ad8eb30bf35a2aaf07f9473f9168e2da" \
-        # MACOS only has next-to-obsolete Python2 installed.  Install Python 3 from python.org
+            "${ideurl}-windows.zip"
-        wget -q https://www.python.org/ftp/python/3.7.4/python-3.7.4-macosx10.9.pkg
+        mv arduino-$idever arduino-distrib
-        sudo installer -pkg python-3.7.4-macosx10.9.pkg -target /
+    elif [ "${RUNNER_OS-}" = "macOS" ]; then
-        # Install the Python3 certificates, because SSL connections fail w/o them and of course they aren't installed by default.
+        fetch_and_unpack "arduino-macos.zip" \
-        ( cd "/Applications/Python 3.7/" && sudo "./Install Certificates.command" )
+            "053b0c1e70da9176680264e40fcb9502f45ca5a879aeb8b6f71282b38bfdb87c63ebc6b88e35ea70a73720ad439d828cc8cb110e4c6ab07357126a36ee396325" \
+            "${ideurl}-macosx.zip"
         # Hack to place arduino-builder in the same spot as sane OSes
-        test -r ${core_path}/tools/dist/arduino-macos.zip || wget -q -O ${core_path}/tools/dist/arduino-macos.zip "${ideurl}-macosx.zip"
-        unzip -q ${core_path}/tools/dist/arduino-macos.zip
         mv Arduino.app arduino-distrib
         mv arduino-distrib/Contents/Java/* arduino-distrib/.
     else
-        test -r ${core_path}/tools/dist/arduino-linux.tar.xz || wget -q -O ${core_path}/tools/dist/arduino-linux.tar.xz "${ideurl}-linux64.tar.xz"
+        fetch_and_unpack "arduino-linux.tar.xz" \
-        tar xf ${core_path}/tools/dist/arduino-linux.tar.xz
+            "9328abf8778200019ed40d4fc0e6afb03a4cee8baaffbcea7dd3626477e14243f779eaa946c809fb153a542bf2ed60cf11a5f135c91ecccb1243c1387be95328" \
-        mv arduino-${idever} arduino-distrib
+            "${ideurl}-linux64.tar.xz"
-    fi
+        mv arduino-$idever arduino-distrib
-    mv arduino-distrib $ide_path
-    cd $ide_path/hardware
-    mkdir esp8266com
-    cd esp8266com
-    if [ "$WINDOWS" = "1" ]; then
-        cp -a $core_path esp8266
-    else
-        ln -s $core_path esp8266
     fi
+
+    mv arduino-distrib "$ide_path"
+    popd
+}
+
+function install_core()
+{
+    local core_path=$1
+    local hardware_core_path=$2
+    local debug=$3
+
+    pushd "${core_path}"
+
     local debug_flags=""
     if [ "$debug" = "debug" ]; then
-        debug_flags="-DDEBUG_ESP_PORT=Serial -DDEBUG_ESP_SSL -DDEBUG_ESP_TLS_MEM -DDEBUG_ESP_HTTP_CLIENT -DDEBUG_ESP_HTTP_SERVER -DDEBUG_ESP_CORE -DDEBUG_ESP_WIFI -DDEBUG_ESP_HTTP_UPDATE -DDEBUG_ESP_UPDATER -DDEBUG_ESP_OTA -DDEBUG_ESP_OOM"
+        debug_flags="-DDEBUG_ESP_PORT=Serial -DDEBUG_ESP_SSL -DDEBUG_ESP_TLS_MEM"\
+" -DDEBUG_ESP_HTTP_CLIENT -DDEBUG_ESP_HTTP_SERVER -DDEBUG_ESP_CORE -DDEBUG_ESP_WIFI"\
+" -DDEBUG_ESP_HTTP_UPDATE -DDEBUG_ESP_UPDATER -DDEBUG_ESP_OTA -DDEBUG_ESP_OOM"
     fi
-    # Set custom warnings for all builds (i.e. could add -Wextra at some point)
+
-    echo "compiler.c.extra_flags=-Wall -Wextra -Werror $debug_flags" > esp8266/platform.local.txt
+    # Set our custom warnings for all builds
-    echo "compiler.cpp.extra_flags=-Wall -Wextra -Werror $debug_flags" >> esp8266/platform.local.txt
+    { echo "compiler.c.extra_flags=-Wall -Wextra -Werror $debug_flags";
-    echo "mkbuildoptglobals.extra_flags=--ci --cache_core" >> esp8266/platform.local.txt
+      echo "compiler.cpp.extra_flags=-Wall -Wextra -Werror $debug_flags";
+      echo "mkbuildoptglobals.extra_flags=--ci --cache_core"; } \
+          > platform.local.txt
     echo -e "\n----platform.local.txt----"
-    cat esp8266/platform.local.txt
+    cat platform.local.txt
     echo -e "\n----\n"
-    cd esp8266/tools
+
+    pushd tools
     python3 get.py -q
-    if [ "$WINDOWS" = "1" ]; then
+
-        # Because the symlinks don't work well under Win32, we need to add the path to this copy, not the original...
+    popd
-        relbin=$(realpath $PWD/xtensa-lx106-elf/bin)
+    popd
-        export PATH="$ide_path:$relbin:$PATH"
+
+    local core_dir
+    core_dir=$(dirname "$hardware_core_path")
+    mkdir -p "$core_dir"
+
+    if [ "${RUNNER_OS-}" = "Windows" ]; then
+        cp -a "$core_path" "${core_dir}/esp8266"
     else
-        export PATH="$ide_path:$core_path/tools/xtensa-lx106-elf/bin:$PATH"
+        ln -s "$core_path" "$hardware_core_path"
     fi
 }
 
 function install_arduino()
 {
+    echo ::group::Install arduino
     local debug=$1
-    # Install Arduino IDE and required libraries
+
-    echo -e "travis_fold:start:sketch_test_env_prepare"
+    test -d "$ESP8266_ARDUINO_IDE" \
-    cd $TRAVIS_BUILD_DIR
+        || install_ide "$ESP8266_ARDUINO_BUILD_DIR" "$ESP8266_ARDUINO_IDE"
-    install_ide $HOME/arduino_ide $TRAVIS_BUILD_DIR $debug
+
-    cd $TRAVIS_BUILD_DIR
+    local hardware_core_path="$ESP8266_ARDUINO_HARDWARE/esp8266com/esp8266"
-    install_libraries
+    test -d "$hardware_core_path" \
-    echo -e "travis_fold:end:sketch_test_env_prepare"
+        || install_core "$ESP8266_ARDUINO_BUILD_DIR" "$hardware_core_path" "$debug"
+
+    install_libraries "$ESP8266_ARDUINO_BUILD_DIR" "$ESP8266_ARDUINO_LIBRARIES"
+
+    echo ::endgroup::
+}
+
+function arduino_lwip_menu_option()
+{
+    case $1 in
+    "default")
+        echo "lm2f"
+        ;;
+    "IPv6")
+        echo "lm6f"
+        ;;
+    esac
 }
 
 function build_sketches_with_arduino()
 {
     local build_mod=$1
     local build_rem=$2
-    local lwip=$3
 
-    # Compile sketches
+    local lwip
-    echo -e "travis_fold:start:sketch_test"
+    lwip=$(arduino_lwip_menu_option $3)
-    build_sketches $HOME/arduino_ide $TRAVIS_BUILD_DIR/libraries "-l $HOME/Arduino/libraries" $build_mod $build_rem $lwip
-    echo -e "travis_fold:end:sketch_test"
 
-    # Generate size report
+    build_sketches "$ESP8266_ARDUINO_BUILD_DIR" \
-    echo -e "travis_fold:start:size_report"
+        "$ESP8266_ARDUINO_IDE" \
-    cat size.log
+        "$ESP8266_ARDUINO_HARDWARE" \
-    echo -e "travis_fold:end:size_report"
+        "$ESP8266_ARDUINO_LIBRARIES" \
+        "$build_mod" "$build_rem" "$lwip"
+    step_summary "Size report" "$cache_dir/size.log"
 }
 
+function install_platformio()
+{
+    echo ::group::Install PlatformIO
 
-set -e
+    local board=$1
 
-if [ -z "$TRAVIS_BUILD_DIR" ]; then
+    pushd $ESP8266_ARDUINO_BUILD_DIR/tools
-    echo "TRAVIS_BUILD_DIR is not set, trying to guess:"
+    python3 get.py -q
-    pushd $(dirname $0)/../ > /dev/null
+    popd
-    TRAVIS_BUILD_DIR=$PWD
+
-    popd > /dev/null
+    # we should reference our up-to-date build tools
-    echo "TRAVIS_BUILD_DIR=$TRAVIS_BUILD_DIR"
+    # ref. https://docs.platformio.org/en/latest/core/userguide/pkg/cmd_install.html
+    pio pkg install --global --skip-dependencies --platform "https://github.com/platformio/platform-espressif8266.git"
+
+    local framework_symlink="framework-arduinoespressif8266 @ symlink://${ESP8266_ARDUINO_BUILD_DIR}"
+    local toolchain_symlink="toolchain-xtensa @ symlink://${ESP8266_ARDUINO_BUILD_DIR}/tools/xtensa-lx106-elf/"
+
+    # pre-generate config; pio-ci with multiple '-O' replace each other instead of appending to the same named list
+    # (and, it is much nicer to write this instead of a multi-line cmdline with several large strings)
+    cat <<EOF > $cache_dir/platformio.ini
+[env:$board]
+platform = espressif8266
+board = $board
+framework = arduino
+platform_packages =
+    ${framework_symlink}
+    ${toolchain_symlink}
+EOF
+
+    # Install dependencies:
+    # - esp8266/examples/ConfigFile
+    pio pkg install --global --library "ArduinoJson@^6.11.0"
+
+    echo ::endgroup::
+}
+
+function build_sketches_with_platformio()
+{
+    local build_mod=$1
+    local build_rem=$2
+    local testcnt=0
+
+    for sketch in $ESP8266_ARDUINO_SKETCHES; do
+        testcnt=$(( ($testcnt + 1) % $build_mod ))
+        if [ $testcnt -ne $build_rem ]; then
+            continue  # Not ours to do
+        fi
+
+        local sketchdir
+        sketchdir=$(dirname $sketch)
+
+        local sketchdirname
+        sketchdirname=$(basename $sketchdir)
+
+        local sketchname
+        sketchname=$(basename $sketch)
+
+        local skip
+        skip=$(skip_sketch "$sketch" "$sketchname" "$sketchdir" "$sketchdirname")
+        if [ -n "$skip" ]; then
+            echo "$skip"
+            continue
+        fi
+
+        echo ::group::Building $sketch
+
+        local result
+        time pio ci \
+            --verbose \
+            --project-conf $cache_dir/platformio.ini \
+            $sketchdir >$cache_dir/build.log 2>&1 \
+            && result=0 || result=1
+
+        if [ $result -ne 0 ]; then
+            echo ::error::Build failed for $sketch
+            cat "$cache_dir/build.log"
+            echo ::endgroup::
+            return $result
+        fi
+
+        echo ::endgroup::
+    done
+}
+
+if [ -z "${ESP8266_ARDUINO_BUILD_DIR-}" ]; then
+    ESP8266_ARDUINO_BUILD_DIR=$(git rev-parse --show-toplevel)
+    echo "Using ESP8266_ARDUINO_BUILD_DIR=$ESP8266_ARDUINO_BUILD_DIR"
 fi

tests/debug.sh

@@ -1,18 +0,0 @@
-#!/usr/bin/env bash
-
-cache_dir=$(mktemp -d)
-
-source "$TRAVIS_BUILD_DIR"/tests/common.sh
-
-if [ "$BUILD_PARITY" = "even" ]; then
-    mod=2
-    rem=0
-elif [ "$BUILD_PARITY" = "odd" ]; then
-    mod=2
-    rem=1
-fi
-
-install_arduino debug
-build_sketches_with_arduino "$mod" "$rem" lm2f
-
-rm -rf "$cache_dir"

tests/debug6.sh

@@ -1,21 +0,0 @@
-#!/usr/bin/env bash
-
-cache_dir=$(mktemp -d)
-
-source "$TRAVIS_BUILD_DIR"/tests/common.sh
-
-if [ -z "$BUILD_PARITY" ]; then
-    mod=1
-    rem=0
-elif [ "$BUILD_PARITY" = "even" ]; then
-    mod=2
-    rem=0
-elif [ "$BUILD_PARITY" = "odd" ]; then
-    mod=2
-    rem=1
-fi
-
-install_arduino debug
-build_sketches_with_arduino "$mod" "$rem" lm6f
-
-rm -rf "$cache_dir"

tests/platformio.sh

@@ -1,83 +0,0 @@
-#!/usr/bin/env bash
-
-cache_dir=$(mktemp -d)
-
-source "$TRAVIS_BUILD_DIR"/tests/common.sh
-
-function install_platformio()
-{
-    pip3 install -U platformio
-    platformio platform install "https://github.com/platformio/platform-espressif8266.git"
-    # Overwrite toolchain with this PR's toolset.  Probably better way to do this
-    ( cd $TRAVIS_BUILD_DIR/tools && python3 get.py -q )
-    mv $TRAVIS_BUILD_DIR/tools/xtensa-lx106-elf ~/.platformio/packages/toolchain-xtensa-latest
-    mv ~/.platformio/packages/toolchain-xtensa/package.json ~/.platformio/packages/toolchain-xtensa/.piopm ~/.platformio/packages/toolchain-xtensa-latest/
-    python -c "import json; import os; fp=open(os.path.expanduser('~/.platformio/platforms/espressif8266/platform.json'), 'r+'); data=json.load(fp); data['packages']['framework-arduinoespressif8266']['version'] = '*'; del data['packages']['framework-arduinoespressif8266']['owner'];fp.seek(0); fp.truncate(); json.dump(data, fp); fp.close()"
-    ln -sf $TRAVIS_BUILD_DIR ~/.platformio/packages/framework-arduinoespressif8266
-    # Install dependencies:
-    # - esp8266/examples/ConfigFile
-    pio lib --global install "ArduinoJson@^6.11.0"
-}
-
-function build_sketches_with_platformio()
-{
-    set +e
-    local srcpath=$1
-    local build_arg=$2
-    local build_mod=$3
-    local build_rem=$4
-    local sketches=$(find $srcpath -name *.ino | sort)
-    local testcnt=0
-    for sketch in $sketches; do
-        testcnt=$(( ($testcnt + 1) % $build_mod ))
-        if [ $testcnt -ne $build_rem ]; then
-            continue  # Not ours to do
-        fi
-        local sketchdir=$(dirname $sketch)
-        local sketchdirname=$(basename $sketchdir)
-        local sketchname=$(basename $sketch)
-        if [[ "${sketchdirname}.ino" != "$sketchname" ]]; then
-            echo "Skipping $sketch, because it is not the main sketch file";
-            continue
-        fi;
-        if [[ -f "$sketchdir/.test.skip" ]]; then
-            echo -e "\n ------------ Skipping $sketch ------------ \n";
-            continue
-        fi
-        if [[ $(skip_ino $sketch) = 1 ]]; then
-            echo -e "\n ------------ Skipping $sketch ------------ \n";
-            continue
-        fi
-        local build_cmd="pio ci $sketchdir $build_arg"
-        echo -e "\n ------------ Building $sketch ------------ \n";
-        echo "$build_cmd"
-        time ($build_cmd >build.log)
-        local result=$?
-        if [ $result -ne 0 ]; then
-            echo "Build failed ($1)"
-            echo "Build log:"
-            cat build.log
-            set -e
-            return $result
-        fi
-        rm build.log
-    done
-    set -e
-}
-
-if [ -z "$BUILD_PARITY" ]; then
-    mod=1
-    rem=0
-elif [ "$BUILD_PARITY" = "even" ]; then
-    mod=2
-    rem=0
-elif [ "$BUILD_PARITY" = "odd" ]; then
-    mod=2
-    rem=1
-fi
-
-install_platformio
-build_sketches_with_platformio "$TRAVIS_BUILD_DIR"/libraries "--board nodemcuv2 --verbose" "$mod" "$rem"
-
-rm -rf "$cache_dir"
-

tests/run_CI_locally.sh

@@ -82,34 +82,41 @@ done
 git submodule update --init
 
 export HOME="${TMPCI}"
-export TRAVIS_BUILD_DIR="${TMPCI}"
+export ESP8266_ARDUINO_BUILD_DIR="${TMPCI}"
 export BUILD_TYPE="$BUILD_TYPE"
 
 if [ "$BUILD_TYPE" = "build" ]; then
     tests/build.sh
+
 elif [ "$BUILD_TYPE" = "build_even" ]; then
-    BUILD_PARITY=even tests/build.sh
+    tests/build.sh even
+
 elif [ "$BUILD_TYPE" = "build_odd" ]; then
-    BUILD_PARITY=odd tests/build.sh
+    tests/build.sh odd
 
 elif [ "$BUILD_TYPE" = "debug_even" ]; then
-    BUILD_PARITY=even tests/debug.sh
+    env ESP8266_ARDUINO_DEBUG=debug tests/build.sh even
+
 elif [ "$BUILD_TYPE" = "debug_odd" ]; then
-    BUILD_PARITY=odd tests/debug.sh
+    env ESP8266_ARDUINO_DEBUG=debug tests/build.sh odd
 
 elif [ "$BUILD_TYPE" = "build6" ]; then
-    tests/build6.sh
+    env ESP8266_ARDUINO_LWIP=lm6f tests/build.sh
+
 elif [ "$BUILD_TYPE" = "build6_even" ]; then
-    BUILD_PARITY=even tests/build6.sh
+    env ESP8266_ARDUINO_LWIP=lm6f tests/build.sh even
+
 elif [ "$BUILD_TYPE" = "build6_odd" ]; then
-    BUILD_PARITY=odd tests/build6.sh
+    env ESP8266_ARDUINO_LWIP=lm6f tests/build.sh odd
 
 elif [ "$BUILD_TYPE" = "platformio" ]; then
-    tests/platformio.sh
+    env ESP8266_ARDUINO_BUILDER=platformio tests/build.sh
+
 elif [ "$BUILD_TYPE" = "platformio_even" ]; then
-    BUILD_PARITY=even tests/platformio.sh
+    env ESP8266_ARDUINO_BUILDER=platformio tests/build.sh even
+
 elif [ "$BUILD_TYPE" = "platformio_odd" ]; then
-    BUILD_PARITY=odd tests/platformio.sh
+    env ESP8266_ARDUINO_BUILDER=platformio tests/build.sh odd
 
 elif [ "$BUILD_TYPE" = host ]; then
     tests/ci/host_test.sh

tools/build.py

@@ -30,34 +30,25 @@ import tempfile
 import shutil
 
 
-# Arduino-builder needs forward-slash paths for passed in params or it cannot
+def compile(tmp_dir, sketch, cache, ide_path, f, args):
-# launch the needed toolset.
-def windowsize_paths(l):
-    """Convert forward-slash paths to backslash paths referenced from C:"""
-    out = []
-    for i in l:
-        if i.startswith('/'):
-            i = 'C:' + i
-        out += [i.replace('/', '\\')]
-    return out
-
-def compile(tmp_dir, sketch, cache, tools_dir, hardware_dir, ide_path, f, args):
     cmd = []
-    cmd += [ide_path + '/arduino-builder']
+    cmd += [os.path.join(ide_path, 'arduino-builder')]
     cmd += ['-compile', '-logger=human']
     cmd += ['-build-path', tmp_dir]
-    cmd += ['-tools', ide_path + '/tools-builder']
     if cache != "":
         cmd += ['-build-cache', cache ]
-    if args.library_path:
+
-        for lib_dir in args.library_path:
+    cmd += ['-tools', os.path.join(ide_path, 'tools-builder')]
-            cmd += ['-libraries', lib_dir]
+    cmd += ['-hardware', os.path.join(ide_path, 'hardware')]
-    cmd += ['-hardware', ide_path + '/hardware']
+
-    if args.hardware_dir:
+    flag_paths = [
-        for hw_dir in args.hardware_dir:
+        ['-tools', args.tool_path],
-            cmd += ['-hardware', hw_dir]
+        ['-libraries', args.library_path],
-    else:
+        ['-hardware', args.hardware_path]]
-        cmd += ['-hardware', hardware_dir]
+    for flag, paths in flag_paths:
+        for path in paths or []:
+            cmd += [flag, path]
+
     # Debug=Serial,DebugLevel=Core____
     fqbn = '-fqbn=esp8266com:esp8266:{board_name}:' \
             'xtal={cpu_freq},' \
@@ -72,16 +63,13 @@ def compile(tmp_dir, sketch, cache, tools_dir, hardware_dir, ide_path, f, args):
     if args.waveform_phase:
         fqbn += ',waveform=phase'
     cmd += [fqbn]
-    cmd += ['-built-in-libraries', ide_path + '/libraries']
+    cmd += ['-built-in-libraries', os.path.join(ide_path, 'libraries')]
     cmd += ['-ide-version=10802']
     cmd += ['-warnings={warnings}'.format(**vars(args))]
     if args.verbose:
         cmd += ['-verbose']
     cmd += [sketch]
 
-    if platform.system() == "Windows":
-        cmd = windowsize_paths(cmd)
-
     if args.verbose:
         print('Building: ' + " ".join(cmd), file=f)
 
@@ -95,12 +83,14 @@ def parse_args():
                         action='store_true')
     parser.add_argument('-i', '--ide_path', help='Arduino IDE path')
     parser.add_argument('-p', '--build_path', help='Build directory')
+    parser.add_argument('-t', '--tool_path', help='Additional tool path',
+                        action='append')
+    parser.add_argument('-d', '--hardware_path', help='Additional hardware path',
+                        action='append')
     parser.add_argument('-l', '--library_path', help='Additional library path',
                         action='append')
-    parser.add_argument('-d', '--hardware_dir', help='Additional hardware path',
-                        action='append')
     parser.add_argument('-b', '--board_name', help='Board name', default='generic')
-    parser.add_argument('-s', '--flash_size', help='Flash size', default='512K64',
+    parser.add_argument('-s', '--flash_size', help='Flash size', default='4M1M',
                         choices=['512K0', '512K64', '1M512', '4M1M', '4M3M'])
     parser.add_argument('-f', '--cpu_freq', help='CPU frequency', default=80,
                         choices=[80, 160], type=int)
@@ -112,7 +102,7 @@ def parse_args():
                         default='none', choices=['none', 'all', 'more'])
     parser.add_argument('-o', '--output_binary', help='File name for output binary')
     parser.add_argument('-k', '--keep', action='store_true',
-                        help='Don\'t delete temporary build directory')
+                        help="Don't delete temporary build directory")
     parser.add_argument('--flash_freq', help='Flash frequency', default=40,
                         type=int, choices=[40, 80])
     parser.add_argument('--debug_port', help='Debug port',
@@ -121,6 +111,9 @@ def parse_args():
                         help='Select waveform locked on phase')
     parser.add_argument('--debug_level', help='Debug level')
     parser.add_argument('--build_cache', help='Build directory to cache core.a', default='')
+    parser.add_argument('--log', nargs='?', help='Redirect output to a file',
+                        type=argparse.FileType('w'),
+                        default=sys.stdout)
     parser.add_argument('sketch_path', help='Sketch file path')
     return parser.parse_args()
 
@@ -128,25 +121,17 @@ def main():
     args = parse_args()
 
     ide_path = args.ide_path
-    if not ide_path:
-        ide_path = os.environ.get('ARDUINO_IDE_PATH')
-        if not ide_path:
-            print("Please specify Arduino IDE path via --ide_path option"
-                  "or ARDUINO_IDE_PATH environment variable.", file=sys.stderr)
-            return 2
-
     sketch_path = args.sketch_path
     tmp_dir = args.build_path
+
     created_tmp_dir = False
     if not tmp_dir:
         tmp_dir = tempfile.mkdtemp()
         created_tmp_dir = True
 
-    tools_dir = os.path.dirname(os.path.realpath(__file__)) + '/../tools'
+    file = os.path.realpath(__file__)
-    # this is not the correct hardware folder to add.
-    hardware_dir = os.path.dirname(os.path.realpath(__file__)) + '/../cores'
 
-    output_name = tmp_dir + '/' + os.path.basename(sketch_path) + '.bin'
+    output_name = os.path.join(tmp_dir, f'{os.path.basename(sketch_path)}.bin')
 
     if args.verbose:
         print("Sketch: ", sketch_path)
@@ -154,12 +139,7 @@ def main():
         print("Cache dir: ", args.build_cache)
         print("Output: ", output_name)
 
-    if args.verbose:
+    res = compile(tmp_dir, sketch_path, args.build_cache, ide_path, args.log, args)
-        f = sys.stdout
-    else:
-        f = open(tmp_dir + '/build.log', 'w')
-
-    res = compile(tmp_dir, sketch_path, args.build_cache, tools_dir, hardware_dir, ide_path, f, args)
     if res != 0:
         return res