From 198054bda47eec1d2d4abd177fd5ac6318a8fbfd Mon Sep 17 00:00:00 2001 From: Alexander Presnyakov Date: Wed, 30 Apr 2025 03:06:33 +0000 Subject: [PATCH] Much more cmake processes when sccache is enabled (because build becomes IO bound task) --- .drone.jsonnet | 6 +++++- build/bootstrap_mcs.sh | 5 +++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.drone.jsonnet b/.drone.jsonnet index a060cf5b9..7a9eb337e 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -718,6 +718,7 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise') ], }, { + // Get the copy of compile cache from the nightly build from S3 name: 'get-local-sccache', depends_on: ['submodules'], image: 'amazon/aws-cli', @@ -737,7 +738,10 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise') 'if aws s3 cp s3://cspkg/nightly-sccache/stable-23.10-' + server + '-' + arch + '-' + std.strReplace(platform, ":", "-") + '/sccache.tar.zst /tmp/sccache.tar.zst; then', ' echo "SCCache archive found. Extracting..."', ' tar -I zstd -xf /tmp/sccache.tar.zst -C /mdb/sccache', - ' ls /mdb/sccache/ && mv /mdb/sccache/mdb/sccache /mdb/sccache_inner && rm -rf /mdb/sccache/ && mv /mdb/sccache_inner /mdb/sccache', # TODO remove + + // TODO remove when all correct caches are uploaded + ' ls /mdb/sccache/; if [ -f /mdb/sccache/mdb ]; then mv /mdb/sccache/mdb/sccache /mdb/sccache_inner && rm -rf /mdb/sccache/ && mv /mdb/sccache_inner /mdb/sccache; fi', + ' echo "SCCache successfully downloaded and extracted."', ' ls /mdb/sccache', 'else', diff --git a/build/bootstrap_mcs.sh b/build/bootstrap_mcs.sh index 7f4a8fd12..364684fc6 100755 --- a/build/bootstrap_mcs.sh +++ b/build/bootstrap_mcs.sh @@ -317,6 +317,11 @@ construct_cmake_flags() { if [[ $SCCACHE = true ]]; then warn "Use sccache" + # If sccache works well, the build becomes an IO bound task instead of CPU bound + # It happens because most of the time is spent is hashing the input files + # So we can run more jobs in parallel + CPUS=$((CPUS * 3)) + message "Adjusted the number of CPUs for parallel build to $CPUS (multiplied by 3 to reduce IO wait)" MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache" fi