Speed up CI builds with caching hacks (#5539)

tests/common.sh

@@ -43,7 +43,7 @@ function build_sketches()
     local build_rem=$5
     local lwip=$6
     mkdir -p $build_dir
-    local build_cmd="python tools/build.py -b generic -v -w all -s 4M1M -v -k -p $PWD/$build_dir -n $lwip $build_arg "
+    local build_cmd="python tools/build.py -b generic -v -w all -s 4M1M -v -k --build_cache $cache_dir -p $PWD/$build_dir -n $lwip $build_arg "
     local sketches=$(find $srcpath -name *.ino | sort)
     print_size_info >size.log
     export ARDUINO_IDE_PATH=$arduino
@@ -53,7 +53,21 @@ function build_sketches()
         if [ $testcnt -ne $build_rem ]; then
             continue  # Not ours to do
         fi
-        rm -rf $build_dir/*
+
+        if [ -e $cache_dir/core/*.a ]; then
+            # We need to preserve the build.options.json file and replace the last .ino
+            # with this sketch's ino file, or builder will throw everything away.
+	    sed -i "s,^.*sketchLocation.*$, \"sketchLocation\": \"$sketch\"\,,g" $build_dir/build.options.json
+            # Set the time of the cached core.a file to the future so the GIT header
+            # we regen won't cause the builder to throw it out and rebuild from scratch.
+            touch -d 'now + 1 day' $cache_dir/core/*.a
+        fi
+
+        # Clear out the last built sketch, map, elf, bin files, but leave the compiled
+        # objects in the core and libraries available for use so we don't need to rebuild
+        # them each sketch.
+        rm -rf $build_dir/sketch $build_dir/*.bin $build_dir/*.map $build_dir/*.elf
+
         local sketchdir=$(dirname $sketch)
         local sketchdirname=$(basename $sketchdir)
         local sketchname=$(basename $sketch)
@@ -221,6 +235,8 @@ if [ -z "$TRAVIS_BUILD_DIR" ]; then
     echo "TRAVIS_BUILD_DIR=$TRAVIS_BUILD_DIR"
 fi
 
+cache_dir=$(mktemp -d)
+
 if [ "$BUILD_TYPE" = "build" ]; then
     install_arduino nodebug
     build_sketches_with_arduino 1 0 lm2f
@@ -259,6 +275,8 @@ elif [ "$BUILD_TYPE" = "platformio_odd" ]; then
     build_sketches_with_platformio $TRAVIS_BUILD_DIR/libraries "--board nodemcuv2 --verbose" 2 1
 else
     echo "BUILD_TYPE not set or invalid"
+    rm -rf $cache_dir
     exit 1
 fi
 
+rm -rf $cache_dir

tools/build.py

@@ -29,11 +29,13 @@ import subprocess
 import tempfile
 import shutil
 
-def compile(tmp_dir, sketch, tools_dir, hardware_dir, ide_path, f, args):
+def compile(tmp_dir, sketch, cache, tools_dir, hardware_dir, ide_path, f, args):
     cmd = ide_path + '/arduino-builder '
     cmd += '-compile -logger=human '
     cmd += '-build-path "' + tmp_dir + '" '
     cmd += '-tools "' +  ide_path + '/tools-builder" '
+    if cache != "":
+        cmd += '-build-cache "' + cache + '" '
     if args.library_path:
         for lib_dir in args.library_path:
             cmd += '-libraries "' + lib_dir + '" '
@@ -98,6 +100,7 @@ def parse_args():
     parser.add_argument('--debug_port', help='Debug port',
                         choices=['Serial', 'Serial1'])
     parser.add_argument('--debug_level', help='Debug level')
+    parser.add_argument('--build_cache', help='Build directory to cache core.a', default='')
     parser.add_argument('sketch_path', help='Sketch file path')
     return parser.parse_args()
 
@@ -127,6 +130,7 @@ def main():
     if args.verbose:
         print("Sketch: ", sketch_path)
         print("Build dir: ", tmp_dir)
+        print("Cache dir: ", args.build_cache)
         print("Output: ", output_name)
 
     if args.verbose:
@@ -134,7 +138,7 @@ def main():
     else:
         f = open(tmp_dir + '/build.log', 'w')
 
-    res = compile(tmp_dir, sketch_path, tools_dir, hardware_dir, ide_path, f, args)
+    res = compile(tmp_dir, sketch_path, args.build_cache, tools_dir, hardware_dir, ide_path, f, args)
     if res != 0:
         return res