1
0
mirror of https://github.com/esp8266/Arduino.git synced 2025-07-30 16:24:09 +03:00

Rework CI workflows (#8688)

- split workflows into separate files to trigger by path
  this should help out documentation and boards / eboot / pkg files
  updates, since those *wont* trigger usual build stuff anymore
- build*.sh whatever merged into just common.sh and build.sh
  trigger different parity builds, mod % rem and allow to set .ino list
  through the environment variable
- removes unnecessary temporary files, try to use more pipes
  move remaining ones into cache dir instead of PWD
- remove legacy TRAVIS env vars, use ESP8266_ARDUINO prefix for config
- remove Windows path workarounds
- hardware/ and ide/ directories are set through envionment
  do not force specific paths, simplify builds on local machine
- sketch list is set through environment. expicit paths for Windows and
  macOS builders. platformio also gets a real shuffled list instead of
  mod and rem magic numbers
- detect root of the repo through git cli, not base{name,dir} or relative paths
This commit is contained in:
Max Prokhorov
2022-10-31 21:13:40 +03:00
committed by GitHub
parent 2360d1cff1
commit 0aab6ecee2
28 changed files with 925 additions and 772 deletions

45
.github/workflows/build-host.yml vendored Normal file
View File

@ -0,0 +1,45 @@
# Run host test suite under valgrind for runtime checking of code.
# Also, a quick test that the mocking builds work at all
name: Build on host OS
on:
pull_request:
paths-ignore:
- "boards.txt"
- "package/**"
- "tools/boards.txt.py"
- 'doc/**'
permissions:
contents: read
jobs:
host-tests:
name: Tests
runs-on: ubuntu-latest
defaults:
run:
shell: bash
steps:
- uses: actions/checkout@v3
with:
submodules: true
- run: |
sudo apt update
sudo apt install valgrind lcov
bash ./tests/ci/host_test.sh
mock-check:
name: Mock
runs-on: ubuntu-latest
defaults:
run:
shell: bash
steps:
- uses: actions/checkout@v3
with:
submodules: true
- run: |
cd tests/host
make -j ../../libraries/ESP8266WebServer/examples/FSBrowser/FSBrowser

96
.github/workflows/build-ide.yml vendored Normal file
View File

@ -0,0 +1,96 @@
# Cross-platform builds to ensure our Core and toolchain works
name: Build IDE examples
on:
pull_request:
paths-ignore:
- "boards.txt"
- "package/**"
- "tools/boards.txt.py"
- 'doc/**'
permissions:
contents: read
jobs:
# Examples are built in parallel to avoid CI total job time limitation
build-linux:
name: Linux - LwIP ${{ matrix.lwip }} (${{ matrix.chunk }})
runs-on: ubuntu-latest
defaults:
run:
shell: bash
strategy:
matrix:
lwip: ["default", "IPv6"]
chunk: [0, 1, 2, 3, 4, 5, 6, 7]
steps:
- uses: actions/checkout@v3
with:
submodules: true
- uses: actions/setup-python@v4
with:
python-version: '3.x'
- uses: actions/cache@v3
with:
path: ./tools/dist
key: ${{ runner.os }}-${{ hashFiles('package/package_esp8266com_index.template.json', 'tests/common.sh', 'tests/build.sh') }}
- name: Build Sketches
env:
ESP8266_ARDUINO_BUILDER: "arduino"
ESP8266_ARDUINO_IDE: "${{ runner.temp }}/arduino_ide"
ESP8266_ARDUINO_LWIP: ${{ matrix.lwip }}
run: |
bash ./tests/build.sh 8 ${{ matrix.chunk }}
# Just try to build at least one sketch, since we spend so much time with the matrix above
build-windows:
name: Windows
runs-on: windows-latest
steps:
- uses: actions/checkout@v3
with:
submodules: false
- uses: actions/setup-python@v4
with:
python-version: '3.x'
- uses: actions/cache@v3
with:
path: ./tools/dist
key: ${{ runner.os }}-${{ hashFiles('package/package_esp8266com_index.template.json', 'tests/common.sh', 'tests/build.sh') }}
- name: Build Sketch
env:
ESP8266_ARDUINO_HARDWARE: "${{ runner.temp }}/hardware"
ESP8266_ARDUINO_IDE: "${{ runner.temp }}/arduino_ide"
ESP8266_ARDUINO_SKETCHES: "libraries/esp8266/examples/Blink/Blink.ino"
run: |
bash ./tests/build.sh
build-mac:
name: macOS
runs-on: macOS-latest
defaults:
run:
shell: bash
steps:
- uses: actions/checkout@v3
with:
submodules: false
- uses: actions/setup-python@v4
with:
python-version: '3.x'
- uses: actions/cache@v3
with:
path: ./tools/dist
key: ${{ runner.os }}-${{ hashFiles('package/package_esp8266com_index.template.json', 'tests/common.sh', 'tests/build.sh') }}
- name: Build Sketch
env:
ESP8266_ARDUINO_HARDWARE: "${{ runner.temp }}/hardware"
ESP8266_ARDUINO_IDE: "${{ runner.temp }}/arduino_ide"
ESP8266_ARDUINO_SKETCHES: "libraries/esp8266/examples/Blink/Blink.ino"
run: |
bash ./tests/build.sh

43
.github/workflows/build-platformio.yml vendored Normal file
View File

@ -0,0 +1,43 @@
# We do not distribute any environment settings, so just try to build some sketches
# using the 'master' branch and using the uploaded toolchain version via get.py
# Also, limit the amount of sketches and simply
name: Build examples with PlatformIO
on:
pull_request:
paths-ignore:
- "boards.txt"
- "tools/boards.txt.py"
- 'doc/**'
permissions:
contents: read
jobs:
build-pio:
name: Linux (random sketches)
runs-on: ubuntu-latest
defaults:
run:
shell: bash
steps:
- uses: actions/checkout@v3
with:
submodules: true
- uses: actions/setup-python@v4
with:
python-version: '3.x'
- uses: actions/cache@v3
with:
path: |
tools/dist
~/.cache/pip
~/.platformio/.cache
key: ${{ runner.os }}-${{ hashFiles('package/package_esp8266com_index.template.json', 'tests/common.sh', 'tests/build.sh') }}
- name: Build
env:
ESP8266_ARDUINO_BUILDER: "platformio"
run: |
pip install -U platformio
env ESP8266_ARDUINO_SKETCHES="$(find libraries/ -name '*.ino' | shuf -n 10 -)" bash ./tests/build.sh

View File

@ -0,0 +1,69 @@
# Ensure no manual edits happen to our autogenerated files
name: Check autogenerated files
on:
pull_request:
paths:
- "boards.txt"
- "bootloaders/**"
- "doc/boards.rst"
- "package/**"
- "tools/boards.txt.py"
- "tools/sdk/ld/**"
permissions:
contents: read
jobs:
pkgrefs-check:
name: .json template
runs-on: ubuntu-latest
defaults:
run:
shell: bash
steps:
- uses: actions/checkout@v3
with:
submodules: false
- run: |
bash ./tests/ci/pkgrefs_test.sh
eboot-check:
name: eboot .elf
runs-on: ubuntu-latest
defaults:
run:
shell: bash
steps:
- uses: actions/checkout@v3
with:
submodules: false
- uses: actions/setup-python@v4
with:
python-version: '3.x'
- uses: actions/cache@v3
with:
path: ./tools/dist
key: ${{ runner.os }}-${{ hashFiles('package/package_esp8266com_index.template.json', 'tests/common.sh') }}
- run: |
# ^ reuse toolchain cache from our linux build job
git submodule update --init --remote tools/sdk/uzlib
bash ./tests/ci/eboot_test.sh
boards-txt-check:
name: boards.txt.py
runs-on: ubuntu-latest
defaults:
run:
shell: bash
steps:
- uses: actions/checkout@v3
with:
submodules: false
- uses: actions/setup-python@v4
with:
python-version: '3.x'
- name: Check git-diff result
run: |
bash ./tests/ci/build_boards.sh

30
.github/workflows/documentation.yml vendored Normal file
View File

@ -0,0 +1,30 @@
# Ensure Sphinx can build the documentation properly.
name: Documentation
on:
pull_request:
paths:
- 'doc/**'
permissions:
contents: read
jobs:
documentation:
name: Sphinx build
runs-on: ubuntu-latest
defaults:
run:
shell: bash
steps:
- uses: actions/checkout@v3
with:
submodules: true
- uses: actions/setup-python@v4
with:
python-version: '3.x'
- name: Build documentation
run: |
pip install --user -r doc/requirements.txt
bash ./tests/ci/build_docs.sh

View File

@ -1,330 +0,0 @@
# Run whenever a PR is generated or updated.
# Most jobs check out the code, ensure Python3 is installed, and for build
# tests the ESP8266 toolchain is cached when possible to speed up execution.
name: ESP8266 Arduino CI
on:
pull_request:
permissions:
contents: read
jobs:
# Run 8 parallel jobs for the default build of all examples.
build-linux:
name: Build ${{ matrix.chunk }}
runs-on: ubuntu-latest
defaults:
run:
shell: bash
strategy:
matrix:
chunk: [0, 1, 2, 3, 4, 5, 6, 7]
steps:
- uses: actions/checkout@v3
with:
submodules: true
- uses: actions/setup-python@v4
with:
python-version: '3.x'
- name: Cache Linux toolchain
id: cache-linux
uses: actions/cache@v3
with:
path: ./tools/dist
key: ${{ runner.os }}-${{ hashFiles('package/package_esp8266com_index.template.json', 'tests/common.sh') }}
- name: Build Sketches
env:
TRAVIS_BUILD_DIR: ${{ github.workspace }}
TRAVIS_TAG: ${{ github.ref }}
BUILD_PARITY: custom
mod: 8
rem: ${{ matrix.chunk }}
run: |
bash ./tests/build.sh
# Cover the debug and IPv6 cases by enabling both and running 8 parallel jobs
# over all example code.
build-debug-ipv6:
name: Debug IPv6 ${{ matrix.chunk }}
runs-on: ubuntu-latest
defaults:
run:
shell: bash
strategy:
matrix:
chunk: [0, 1, 2, 3, 4, 5, 6, 7]
steps:
- uses: actions/checkout@v3
with:
submodules: true
- uses: actions/setup-python@v4
with:
python-version: '3.x'
- name: Cache Linux toolchain
id: cache-linux
uses: actions/cache@v3
with:
path: ./tools/dist
key: ${{ runner.os }}-${{ hashFiles('package/package_esp8266com_index.template.json', 'tests/common.sh') }}
- name: Build Sketches
env:
TRAVIS_BUILD_DIR: ${{ github.workspace }}
TRAVIS_TAG: ${{ github.ref }}
BUILD_PARITY: custom
mod: 8
rem: ${{ matrix.chunk }}
run: |
bash ./tests/debug6.sh
# Single build under Windows to ensure the Win toolchain is good.
build-windows:
name: Windows
runs-on: windows-latest
steps:
- uses: actions/checkout@v3
with:
submodules: true
- uses: actions/setup-python@v4
with:
python-version: '3.x'
- name: Cache Windows toolchain
id: cache-windows
uses: actions/cache@v3
with:
path: ./tools/dist
key: ${{ runner.os }}-${{ hashFiles('package/package_esp8266com_index.template.json', 'tests/common.sh') }}
- name: Build Sketch
env:
TRAVIS_BUILD_DIR: ${{ github.workspace }}
TRAVIS_TAG: ${{ github.ref }}
WINDOWS: 1
BUILD_PARITY: custom
mod: 500
rem: 1
run: |
# Windows has python3 already installed, but it's called "python".
# Copy python.exe to the proper name so scripts "just work".
try { Get-Command python3 } catch { copy (get-command python).source (get-command python).source.Replace("python.exe", "python3.exe") }
bash ./tests/build.sh
# Single build under macOS to ensure the Mac toolchain is good.
build-mac:
name: Mac
runs-on: macOS-latest
defaults:
run:
shell: bash
steps:
- uses: actions/checkout@v3
with:
submodules: true
- uses: actions/setup-python@v4
with:
python-version: '3.x'
- name: Cache Mac toolchain
id: cache-mac
uses: actions/cache@v3
with:
path: ./tools/dist
key: ${{ runner.os }}-${{ hashFiles('package/package_esp8266com_index.template.json', 'tests/common.sh') }}
- name: Build Sketch
env:
TRAVIS_BUILD_DIR: ${{ github.workspace }}
TRAVIS_TAG: ${{ github.ref }}
MACOSX: 1
BUILD_PARITY: custom
mod: 500
rem: 1
run: |
bash ./tests/build.sh
# Run a few Platform.IO jobs (not full suite) to check PIO integration.
build-pio:
name: Build Platform.IO
runs-on: ubuntu-latest
defaults:
run:
shell: bash
steps:
- uses: actions/checkout@v3
with:
submodules: true
- uses: actions/setup-python@v4
with:
python-version: '3.x'
- name: Build subset on Platform.IO
env:
TRAVIS_BUILD_DIR: ${{ github.workspace }}
TRAVIS_TAG: ${{ github.ref }}
BUILD_PARITY: custom
mod: 42 # Picked at random to give 4-5 builds and exit.
rem: 13
run: |
sudo apt update
sudo apt install python3-pip python3-setuptools
PATH=/home/runner/.local/bin:$PATH bash ./tests/platformio.sh
# Run host test suite under valgrind for runtime checking of code.
host-tests:
name: Host tests
runs-on: ubuntu-latest
defaults:
run:
shell: bash
steps:
- uses: actions/checkout@v3
with:
submodules: true
- uses: actions/setup-python@v4
with:
python-version: '3.x'
- name: Run host tests
env:
TRAVIS_BUILD_DIR: ${{ github.workspace }}
TRAVIS_TAG: ${{ github.ref }}
run: |
sudo apt update
sudo apt install valgrind lcov
bash ./tests/ci/host_test.sh
# Ensure Sphinx can build the documentation properly.
documentation:
name: Documentation
runs-on: ubuntu-latest
defaults:
run:
shell: bash
steps:
- uses: actions/checkout@v3
with:
submodules: true
- uses: actions/setup-python@v4
with:
python-version: '3.x'
- name: Build documentation
env:
TRAVIS_BUILD_DIR: ${{ github.workspace }}
TRAVIS_TAG: ${{ github.ref }}
run: |
sudo apt update
sudo apt install python3-pip python3-setuptools
# GitHub CI installs pip3 and setuptools outside the path.
# Update the path to include them and run.
PATH=/home/runner/.local/bin:$PATH pip3 install --user -r doc/requirements.txt
PATH=/home/runner/.local/bin:$PATH bash ./tests/ci/build_docs.sh
# Standard Arduino formatting in all the examples
style-check:
name: Style and formatting
runs-on: ubuntu-latest
defaults:
run:
shell: bash
steps:
- uses: actions/checkout@v3
with:
submodules: true
- uses: actions/setup-python@v4
with:
python-version: '3.x'
- name: Style check
env:
LLVM_SNAPSHOT_KEY: "6084F3CF814B57C1CF12EFD515CF4D18AF4F7421"
TRAVIS_BUILD_DIR: ${{ github.workspace }}
TRAVIS_TAG: ${{ github.ref }}
run: |
export GNUPGHOME=$(mktemp -d)
gpg --batch --keyserver keyserver.ubuntu.com --recv-keys "$LLVM_SNAPSHOT_KEY"
gpg --batch --armor --export "$LLVM_SNAPSHOT_KEY" | \
sudo tee /etc/apt/trusted.gpg.d/llvm-snapshot.gpg.asc
gpgconf --kill all
rm -r $GNUPGHOME
echo "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-13 main" | \
sudo tee /etc/apt/sources.list.d/llvm.list
sudo apt update
sudo apt install clang-format-13
pip3 install pyyaml
bash ./tests/ci/style_check.sh
# Quick test that the mocking builds succeed
mock-check:
name: Mock trivial test
runs-on: ubuntu-latest
defaults:
run:
shell: bash
steps:
- uses: actions/checkout@v3
with:
submodules: true
- uses: actions/setup-python@v4
with:
python-version: '3.x'
- name: Mock build
env:
TRAVIS_BUILD_DIR: ${{ github.workspace }}
TRAVIS_TAG: ${{ github.ref }}
run: |
bash ./tests/buildm.sh
# Ensure no manual edits to boards.txt
boards-check:
name: Boards.txt check
runs-on: ubuntu-latest
defaults:
run:
shell: bash
steps:
- uses: actions/checkout@v3
with:
submodules: true
- uses: actions/setup-python@v4
with:
python-version: '3.x'
- name: Cache Linux toolchain
id: cache-linux
uses: actions/cache@v3
with:
path: ./tools/dist
key: ${{ runner.os }}-${{ hashFiles('package/package_esp8266com_index.template.json', 'tests/common.sh') }}
- name: Boards.txt diff
env:
TRAVIS_BUILD_DIR: ${{ github.workspace }}
TRAVIS_TAG: ${{ github.ref }}
run: |
bash ./tests/ci/build_boards.sh
bash ./tests/ci/eboot_test.sh
bash ./tests/ci/pkgrefs_test.sh
# Validate orthography
code-spell:
name: Check spelling
runs-on: ubuntu-latest
defaults:
run:
shell: bash
steps:
- uses: actions/checkout@v3
with:
submodules: true
- name: Run codespell
uses: codespell-project/actions-codespell@master
with:
skip: ./libraries/ESP8266SdFat,./libraries/LittleFS/lib,./tools/pyserial,./tools/sdk,./tools/esptool,./libraries/SoftwareSerial,./libraries/Ethernet,./github/workflows,./libraries/ESP8266HTTPUpdateServer/examples/SecureBearSSLUpdater/SecureBearSSLUpdater.ino,./libraries/esp8266/examples/RTCUserMemory/RTCUserMemory.ino,./libraries/esp8266/examples/StreamString/StreamString.ino,./libraries/ESP8266WiFi/examples/BearSSL_Validation/BearSSL_Validation.ino,./libraries/ESP8266WiFi/examples/BearSSL_Sessions/BearSSL_Sessions.ino,./libraries/ESP8266WebServer/examples/HelloServerBearSSL/HelloServerBearSSL.ino,./libraries/ESP8266WebServer/examples/HttpHashCredAuth/HttpHashCredAuth.ino,./cores/esp8266/spiffs,./tests/device/test_libc/libm_string.c, ./libraries/Netdump/examples/Netdump/Netdump.ino,./libraries/ESP8266WiFi/examples/BearSSL_Server,./cores/esp8266/LwipIntfDev.h
ignore_words_list: ESP8266,esp8266,esp,dout,DOUT,ser,ans

View File

@ -35,9 +35,6 @@ jobs:
package:
name: Update master JSON file
runs-on: ubuntu-latest
defaults:
run:
shell: bash
steps:
- uses: actions/checkout@v3
with:
@ -45,7 +42,6 @@ jobs:
fetch-depth: 0
- name: Deploy updated JSON
env:
TRAVIS_BUILD_DIR: ${{ github.workspace }}
BUILD_TYPE: package
CI_GITHUB_API_KEY: ${{ secrets.GITHUB_TOKEN }}
GHCI_DEPLOY_KEY: ${{ secrets.GHCI_DEPLOY_KEY }}

56
.github/workflows/style-check.yml vendored Normal file
View File

@ -0,0 +1,56 @@
name: Style and syntax checks
on:
pull_request:
permissions:
contents: read
jobs:
# Generic formatting for Core and examples
clang-format:
name: clang-format
runs-on: ubuntu-latest
defaults:
run:
shell: bash
steps:
- uses: actions/checkout@v3
with:
submodules: true
- uses: actions/setup-python@v4
with:
python-version: '3.x'
- name: Style check
env:
LLVM_SNAPSHOT_KEY: "6084F3CF814B57C1CF12EFD515CF4D18AF4F7421"
run: |
export GNUPGHOME=$(mktemp -d)
gpg --batch --keyserver keyserver.ubuntu.com --recv-keys "$LLVM_SNAPSHOT_KEY"
gpg --batch --armor --export "$LLVM_SNAPSHOT_KEY" | \
sudo tee /etc/apt/trusted.gpg.d/llvm-snapshot.gpg.asc
gpgconf --kill all
rm -r $GNUPGHOME
echo "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-13 main" | \
sudo tee /etc/apt/sources.list.d/llvm.list
sudo apt update
sudo apt install clang-format-13
pip3 install pyyaml
bash ./tests/ci/style_check.sh
# Validate orthography
code-spell:
name: codespell
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
with:
submodules: false
- name: Run codespell
uses: codespell-project/actions-codespell@master
with:
skip: ./libraries/ESP8266SdFat,./libraries/LittleFS/lib,./tools/pyserial,./tools/sdk,./tools/esptool,./libraries/SoftwareSerial,./libraries/Ethernet,./github/workflows,./libraries/ESP8266HTTPUpdateServer/examples/SecureBearSSLUpdater/SecureBearSSLUpdater.ino,./libraries/esp8266/examples/RTCUserMemory/RTCUserMemory.ino,./libraries/esp8266/examples/StreamString/StreamString.ino,./libraries/ESP8266WiFi/examples/BearSSL_Validation/BearSSL_Validation.ino,./libraries/ESP8266WiFi/examples/BearSSL_Sessions/BearSSL_Sessions.ino,./libraries/ESP8266WebServer/examples/HelloServerBearSSL/HelloServerBearSSL.ino,./libraries/ESP8266WebServer/examples/HttpHashCredAuth/HttpHashCredAuth.ino,./cores/esp8266/spiffs,./tests/device/test_libc/libm_string.c, ./libraries/Netdump/examples/Netdump/Netdump.ino,./libraries/ESP8266WiFi/examples/BearSSL_Server,./cores/esp8266/LwipIntfDev.h
ignore_words_list: ESP8266,esp8266,esp,dout,DOUT,ser,ans

View File

@ -28,18 +28,24 @@ jobs:
- name: Set GIT tag name
run: |
# Sets an environment variable used in the next steps
TRAVIS_TAG="$(git describe --exact-match --tags)"
echo "TRAVIS_TAG=${TRAVIS_TAG}" >> $GITHUB_ENV
ESP8266_ARDUINO_RELEASE_TAG="$(git describe --exact-match --tags)"
echo "ESP8266_ARDUINO_RELEASE_TAG=${ESP8266_ARDUINO_RELEASE_TAG}" >> $GITHUB_ENV
- name: Build package JSON
env:
TRAVIS_BUILD_DIR: ${{ github.workspace }}
BUILD_TYPE: package
CI_GITHUB_API_KEY: ${{ secrets.GITHUB_TOKEN }}
BUILD_TYPE: package
run: |
bash ./tests/ci/build_package.sh
pip3 install PyGithub
# Create a draft release and upload the ZIP and JSON files.
# This draft is not visible to normal users and needs to be
# updated manually with release notes and published from the
# GitHub web interface.
python3 ./package/upload_release.py --user "$GITHUB_ACTOR" --repo "$GITHUB_REPOSITORY" --token "$CI_GITHUB_API_KEY" --tag "$TRAVIS_TAG" --name "Release $TRAVIS_TAG" --msg "Update the draft with release notes before publishing." package/versions/*/*.zip package/versions/*/package_esp8266com_index.json
bash ./tests/ci/build_package.sh
# Create a draft release and upload the ZIP and JSON files.
# This draft is not visible to normal users and needs to be
# updated manually with release notes and published from the
# GitHub web interface.
pip3 install PyGithub
python3 ./package/upload_release.py \
--user "$GITHUB_ACTOR" \
--repo "$GITHUB_REPOSITORY" \
--token "$CI_GITHUB_API_KEY" \
--tag "$ESP8266_ARDUINO_RELEASE_TAG" \
--name "Release ${ESP8266_ARDUINO_RELEASE_TAG}" \
--msg "Update the draft with release notes before publishing." \
package/versions/*/*.zip package/versions/*/package_esp8266com_index.json