From 1f86311d79a81f6a80a520a053c7697d0fa65bce Mon Sep 17 00:00:00 2001 From: "Earle F. Philhower, III" Date: Tue, 10 Sep 2019 14:50:55 -0700 Subject: [PATCH] Add Win32 build to CI system (#6493) Build a single INO under a Windows VM on Travis to ensure that the Win32 toolchain works properly. In build.py, instead of making a string with spaces and then splitting on " ", just make the list itself with individual parameters. This will allow spaces in paths to be supported properly. --- .travis.yml | 8 ++++++- tests/common.sh | 51 ++++++++++++++++++++++++++++++++++--------- tools/build.py | 57 ++++++++++++++++++++++++++++++++----------------- tools/get.py | 18 ++++++++++++---- 4 files changed, 99 insertions(+), 35 deletions(-) diff --git a/.travis.yml b/.travis.yml index 341db3dbd..e6a2b4012 100644 --- a/.travis.yml +++ b/.travis.yml @@ -107,12 +107,18 @@ jobs: script: $TRAVIS_BUILD_DIR/tests/buildm.sh env: CC=gcc-7 CXX=g++-7 - - name: "MacOS can build sketches" + - name: "Mac OSX can build sketches" os: osx stage: build script: $TRAVIS_BUILD_DIR/tests/build.sh env: MACOSX=1 BUILD_PARITY=custom mod=500 rem=1 + - name: "Windows can build sketches" + os: windows + stage: build + script: $TRAVIS_BUILD_DIR/tests/build.sh + env: WINDOWS=1 BUILD_PARITY=custom mod=500 rem=1 + - name: "Boards" stage: build script: $TRAVIS_BUILD_DIR/tests/ci/build_boards.sh diff --git a/tests/common.sh b/tests/common.sh index 2fe861581..b93bd57ba 100755 --- a/tests/common.sh +++ b/tests/common.sh @@ -44,8 +44,7 @@ function print_size_info() segments[$seg]=$size fi - - done < <(xtensa-lx106-elf-size --format=sysv $elf_file) + done < <(xtensa-lx106-elf-size --format=sysv $elf_file | sed 's/\r//g' ) total_ram=$((${segments[data]} + ${segments[rodata]} + ${segments[bss]})) total_flash=$((${segments[data]} + ${segments[rodata]} + ${segments[text]} + ${segments[irom0text]})) @@ -66,6 +65,10 @@ function build_sketches() local lwip=$6 mkdir -p $build_dir local build_cmd="python3 tools/build.py -b generic -v -w all -s 4M1M -v -k --build_cache $cache_dir -p $PWD/$build_dir -n $lwip $build_arg " + if [ "$WINDOWS" = "1" ]; then + # Paths to the arduino builder need to be / referenced, not our native ones + build_cmd=$(echo $build_cmd --ide_path $arduino | sed 's/ \/c\// \//g' ) # replace '/c/' with '/' + fi local sketches=$(find $srcpath -name *.ino | sort) print_size_info >size.log export ARDUINO_IDE_PATH=$arduino @@ -107,6 +110,14 @@ function build_sketches() fi echo -e "\n ------------ Building $sketch ------------ \n"; # $arduino --verify $sketch; + if [ "$WINDOWS" == "1" ]; then + sketch=$(echo $sketch | sed 's/^\/c//') + # MINGW will try to be helpful and silently convert args that look like paths to point to a spot inside the MinGW dir. This breaks everything. + # http://www.mingw.org/wiki/Posix_path_conversion + # https://stackoverflow.com/questions/7250130/how-to-stop-mingw-and-msys-from-mangling-path-names-given-at-the-command-line#34386471 + export MSYS2_ARG_CONV_EXC="*" + export MSYS_NO_PATHCONV=1 + fi echo "$build_cmd $sketch" time ($build_cmd $sketch >build.log) local result=$? @@ -135,7 +146,7 @@ function install_libraries() pushd $HOME/Arduino/libraries # install ArduinoJson library - { test -r ArduinoJson-v6.11.0.zip || wget https://github.com/bblanchon/ArduinoJson/releases/download/v6.11.0/ArduinoJson-v6.11.0.zip; } && unzip -q ArduinoJson-v6.11.0.zip + { test -r ArduinoJson-v6.11.0.zip || wget -nv https://github.com/bblanchon/ArduinoJson/releases/download/v6.11.0/ArduinoJson-v6.11.0.zip; } && unzip -q ArduinoJson-v6.11.0.zip popd } @@ -145,17 +156,27 @@ function install_ide() local ide_path=$1 local core_path=$2 local debug=$3 - if [ "$MACOSX" = "1" ]; then + if [ "$WINDOWS" = "1" ]; then + # Acquire needed packages from Windows package manager + choco install --no-progress python3 + export PATH="/c/Python37:$PATH" # Ensure it's live from now on... + cp /c/Python37/python.exe /c/Python37/python3.exe + choco install --no-progress unzip + choco install --no-progress sed + #choco install --no-progress golang + test -r arduino-nightly-windows.zip || wget -nv -O arduino-nightly-windows.zip https://www.arduino.cc/download.php?f=/arduino-nightly-windows.zip + unzip -q arduino-nightly-windows.zip + elif [ "$MACOSX" = "1" ]; then # MACOS only has next-to-obsolete Python2 installed. Install Python 3 from python.org wget https://www.python.org/ftp/python/3.7.4/python-3.7.4-macosx10.9.pkg sudo installer -pkg python-3.7.4-macosx10.9.pkg -target / # Install the Python3 certificates, because SSL connections fail w/o them and of course they aren't installed by default. - ( cd "/Applications/Python 3.7/" && sudo "./Install Certificates.command" ) + ( cd "/Applications/Python 3.7/" && sudo "./Install Certificates.command" ) # Hack to place arduino-builder in the same spot as sane OSes test -r arduino.zip || wget -O arduino.zip https://downloads.arduino.cc/arduino-nightly-macosx.zip unzip -q arduino.zip - mv Arduino.app arduino-nightly - mv arduino-nightly/Contents/Java/* arduino-nightly/. + mv Arduino.app arduino-nightly + mv arduino-nightly/Contents/Java/* arduino-nightly/. else test -r arduino.tar.xz || wget -O arduino.tar.xz https://www.arduino.cc/download.php?f=/arduino-nightly-linux64.tar.xz tar xf arduino.tar.xz @@ -164,7 +185,11 @@ function install_ide() cd $ide_path/hardware mkdir esp8266com cd esp8266com - ln -s $core_path esp8266 + if [ "$WINDOWS" = "1" ]; then + cp -a $core_path esp8266 + else + ln -s $core_path esp8266 + fi local debug_flags="" if [ "$debug" = "debug" ]; then debug_flags="-DDEBUG_ESP_PORT=Serial -DDEBUG_ESP_SSL -DDEBUG_ESP_TLS_MEM -DDEBUG_ESP_HTTP_CLIENT -DDEBUG_ESP_HTTP_SERVER -DDEBUG_ESP_CORE -DDEBUG_ESP_WIFI -DDEBUG_ESP_HTTP_UPDATE -DDEBUG_ESP_UPDATER -DDEBUG_ESP_OTA -DDEBUG_ESP_OOM" @@ -176,8 +201,14 @@ function install_ide() cat esp8266/platform.local.txt echo -e "\n----\n" cd esp8266/tools - python3 get.py - export PATH="$ide_path:$core_path/tools/xtensa-lx106-elf/bin:$PATH" + python3 get.py -q + if [ "$WINDOWS" = "1" ]; then + # Because the symlinks don't work well under Win32, we need to add the path to this copy, not the original... + relbin=$(realpath $PWD/xtensa-lx106-elf/bin) + export PATH="$ide_path:$relbin:$PATH" + else + export PATH="$ide_path:$core_path/tools/xtensa-lx106-elf/bin:$PATH" + fi } function install_arduino() diff --git a/tools/build.py b/tools/build.py index 8859bbdfc..efb1409ba 100755 --- a/tools/build.py +++ b/tools/build.py @@ -24,28 +24,42 @@ from __future__ import print_function import sys import os import argparse +import platform import subprocess import tempfile import shutil + +# Arduino-builder needs forward-slash paths for passed in params or it cannot +# launch the needed toolset. +def windowsize_paths(l): + """Convert forward-slash paths to backslash paths referenced from C:""" + out = [] + for i in l: + if i.startswith('/'): + i = 'C:' + i + out += [i.replace('/', '\\')] + return out + def compile(tmp_dir, sketch, cache, tools_dir, hardware_dir, ide_path, f, args): - cmd = ide_path + '/arduino-builder ' - cmd += '-compile -logger=human ' - cmd += '-build-path "' + tmp_dir + '" ' - cmd += '-tools "' + ide_path + '/tools-builder" ' + cmd = [] + cmd += [ide_path + '/arduino-builder'] + cmd += ['-compile', '-logger=human'] + cmd += ['-build-path', tmp_dir] + cmd += ['-tools', ide_path + '/tools-builder'] if cache != "": - cmd += '-build-cache "' + cache + '" ' + cmd += ['-build-cache', cache ] if args.library_path: for lib_dir in args.library_path: - cmd += '-libraries "' + lib_dir + '" ' - cmd += '-hardware "' + ide_path + '/hardware" ' + cmd += ['-libraries', lib_dir] + cmd += ['-hardware', ide_path + '/hardware'] if args.hardware_dir: for hw_dir in args.hardware_dir: - cmd += '-hardware "' + hw_dir + '" ' + cmd += ['-hardware', hw_dir] else: - cmd += '-hardware "' + hardware_dir + '" ' + cmd += ['-hardware', hardware_dir] # Debug=Serial,DebugLevel=Core____ - cmd += '-fqbn=esp8266com:esp8266:{board_name}:' \ + fqbn = '-fqbn=esp8266com:esp8266:{board_name}:' \ 'xtal={cpu_freq},' \ 'FlashFreq={flash_freq},' \ 'FlashMode={flash_mode},' \ @@ -54,20 +68,22 @@ def compile(tmp_dir, sketch, cache, tools_dir, hardware_dir, ide_path, f, args): 'ip={lwIP},' \ 'ResetMethod=nodemcu'.format(**vars(args)) if args.debug_port and args.debug_level: - cmd += 'dbg={debug_port},lvl={debug_level}'.format(**vars(args)) - cmd += ' ' - cmd += '-built-in-libraries "' + ide_path + '/libraries" ' - cmd += '-ide-version=10607 ' - cmd += '-warnings={warnings} '.format(**vars(args)) + fqbn += 'dbg={debug_port},lvl={debug_level}'.format(**vars(args)) + cmd += [fqbn] + cmd += ['-built-in-libraries', ide_path + '/libraries'] + cmd += ['-ide-version=10607'] + cmd += ['-warnings={warnings}'.format(**vars(args))] if args.verbose: - cmd += '-verbose ' - cmd += sketch + cmd += ['-verbose'] + cmd += [sketch] + + if platform.system() == "Windows": + cmd = windowsize_paths(cmd) if args.verbose: - print('Building: ' + cmd, file=f) + print('Building: ' + " ".join(cmd), file=f) - cmds = cmd.split(' ') - p = subprocess.Popen(cmds, stdout=f, stderr=subprocess.STDOUT) + p = subprocess.Popen(cmd, stdout=f, stderr=subprocess.STDOUT) p.wait() return p.returncode @@ -127,6 +143,7 @@ def main(): hardware_dir = os.path.dirname(os.path.realpath(__file__)) + '/../cores' output_name = tmp_dir + '/' + os.path.basename(sketch_path) + '.bin' + if args.verbose: print("Sketch: ", sketch_path) print("Build dir: ", tmp_dir) diff --git a/tools/get.py b/tools/get.py index 9a85ae84b..74b6b510d 100755 --- a/tools/get.py +++ b/tools/get.py @@ -15,6 +15,9 @@ import sys import tarfile import zipfile import re + +verbose = True + if sys.version_info[0] == 3: from urllib.request import urlretrieve else: @@ -38,10 +41,12 @@ def mkdir_p(path): raise def report_progress(count, blockSize, totalSize): - percent = int(count*blockSize*100/totalSize) - percent = min(100, percent) - sys.stdout.write("\r%d%%" % percent) - sys.stdout.flush() + global verbose + if verbose: + percent = int(count*blockSize*100/totalSize) + percent = min(100, percent) + sys.stdout.write("\r%d%%" % percent) + sys.stdout.flush() def unpack(filename, destination): dirname = '' @@ -111,6 +116,11 @@ def identify_platform(): return arduino_platform_names[sys_name][bits] def main(): + global verbose + # Support optional "-q" quiet mode simply + if len(sys.argv) == 2: + if sys.argv[1] == "-q": + verbose = False print('Platform: {0}'.format(identify_platform())) tools_to_download = load_tools_list('../package/package_esp8266com_index.template.json', identify_platform()) mkdir_p(dist_dir)