From cce0f6ab0cb779df0dc732ac5ad42366c62e0c7b Mon Sep 17 00:00:00 2001 From: Leonid Fedorov <79837786+mariadb-LeonidFedorov@users.noreply.github.com> Date: Thu, 20 Jun 2024 13:48:20 +0400 Subject: [PATCH] fix(cgroups)!: Containers memory limits for CI (#3108) (#3209) Limit test containers by memory, fix cgroup path inside the containers by introducing new ugly setting name --------- Co-authored-by: drrtuy Co-authored-by: Roman Nozdrin --- .drone.jsonnet | 36 +++++++++++++++-------- build/bootstrap_mcs.sh | 15 ++++++---- mysql-test/columnstore/basic/disabled.def | 1 + utils/common/cgroupconfigurator.cpp | 4 +-- 4 files changed, 36 insertions(+), 20 deletions(-) diff --git a/.drone.jsonnet b/.drone.jsonnet index 413e4fadc..132b56075 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -251,6 +251,18 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise') if (pkg_format == 'deb') then execInnerDocker('sed -i "s/exit 101/exit 0/g" /usr/sbin/policy-rc.d', dockerImage), + 'echo "Docker CGroups opts here"', + 'ls -al /sys/fs/cgroup/cgroup.controllers || true ', + 'ls -al /sys/fs/cgroup/ || true ', + 'ls -al /sys/fs/cgroup/memory || true', + "docker ps --filter=name=" + dockerImage, + + execInnerDocker('echo "Inner Docker CGroups opts here"', dockerImage), + execInnerDocker('ls -al /sys/fs/cgroup/cgroup.controllers || true', dockerImage), + execInnerDocker('ls -al /sys/fs/cgroup/ || true', dockerImage), + execInnerDocker('ls -al /sys/fs/cgroup/memory || true', dockerImage), + + execInnerDocker('mkdir core', dockerImage), execInnerDocker('chmod 777 core', dockerImage), 'docker cp core_dumps/. ' + dockerImage + ':/', @@ -312,13 +324,14 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise') image: 'docker', volumes: [pipeline._volumes.docker], commands: [ - 'docker run --env OS=' + result + ' --env PACKAGES_URL=' + packages_url + ' --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --name smoke$${DRONE_BUILD_NUMBER} --ulimit core=-1 --privileged --detach ' + img + ' ' + init + ' --unit=basic.target'] + 'docker run --memory 3g --env OS=' + result + ' --env PACKAGES_URL=' + packages_url + ' --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --name smoke$${DRONE_BUILD_NUMBER} --ulimit core=-1 --privileged --detach ' + img + ' ' + init + ' --unit=basic.target'] + prepareTestStage(dockerImage("smoke"), pkg_format, result, true) + [ installEngine(dockerImage("smoke"), pkg_format), 'sleep $${SMOKE_DELAY_SECONDS:-1s}', // start mariadb and mariadb-columnstore services and run simple query execInnerDocker('systemctl start mariadb', dockerImage("smoke")), - execInnerDocker('systemctl start mariadb-columnstore', dockerImage("smoke")), + execInnerDocker("/usr/bin/mcsSetConfig SystemConfig CGroup just_no_group_use_local", dockerImage("smoke")), + execInnerDocker('systemctl restart mariadb-columnstore', dockerImage("smoke")), execInnerDocker('mariadb -e "create database if not exists test; create table test.t1 (a int) engine=Columnstore; insert into test.t1 values (1); select * from test.t1"', dockerImage("smoke")), @@ -340,6 +353,7 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise') }, }, commands: [ + // why do we mount cgroups here, but miss it on other steps? 'docker run --volume /sys/fs/cgroup:/sys/fs/cgroup:ro --env OS=' + result + ' --env PACKAGES_URL=' + packages_url + ' --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --env UCF_FORCE_CONFNEW=1 --name upgrade$${DRONE_BUILD_NUMBER}' + version + ' --ulimit core=-1 --privileged --detach ' + img + ' ' + init + ' --unit=basic.target'] + prepareTestStage(dockerImage('upgrade') + version, pkg_format, result, false) + [ if (pkg_format == 'deb') @@ -375,7 +389,7 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise') MTR_FULL_SUITE: '${MTR_FULL_SUITE:-false}', }, commands: [ - 'docker run --shm-size=500m --env MYSQL_TEST_DIR=' + mtr_path + ' --env OS=' + result + ' --env PACKAGES_URL=' + packages_url + ' --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --name mtr$${DRONE_BUILD_NUMBER} --ulimit core=-1 --privileged --detach ' + img + ' ' + init + ' --unit=basic.target'] + 'docker run --shm-size=500m --memory 8g --env MYSQL_TEST_DIR=' + mtr_path + ' --env OS=' + result + ' --env PACKAGES_URL=' + packages_url + ' --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --name mtr$${DRONE_BUILD_NUMBER} --ulimit core=-1 --privileged --detach ' + img + ' ' + init + ' --unit=basic.target'] + prepareTestStage('mtr$${DRONE_BUILD_NUMBER}', pkg_format, result, true) + [ installEngine(dockerImage("mtr"), pkg_format), 'docker cp mysql-test/columnstore mtr$${DRONE_BUILD_NUMBER}:' + mtr_path + '/suite/', @@ -384,14 +398,11 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise') execInnerDocker("bash -c 'sed -i /ProtectSystem/d $(systemctl show --property FragmentPath mariadb | sed s/FragmentPath=//)'", dockerImage('mtr')), execInnerDocker('systemctl daemon-reload', dockerImage("mtr")), execInnerDocker('systemctl start mariadb', dockerImage("mtr")), + // Set RAM consumption limits to avoid RAM contention b/w mtr and regression steps. + execInnerDocker("/usr/bin/mcsSetConfig SystemConfig CGroup just_no_group_use_local", dockerImage("mtr")), execInnerDocker('mariadb -e "create database if not exists test;"', dockerImage("mtr")), execInnerDocker('systemctl restart mariadb-columnstore', dockerImage("mtr")), - // Set RAM consumption limits to avoid RAM contention b/w mtr and regression steps. - //'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "/usr/bin/mcsSetConfig HashJoin TotalUmMemory 4G"', - //'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "/usr/bin/mcsSetConfig DBBC NumBlocksPct 1G"', - //'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "/usr/bin/mcsSetConfig SystemConfig CGroup $(docker ps --filter=name=mtr$${DRONE_BUILD_NUMBER} --quiet --no-trunc)"', - // delay mtr for manual debugging on live instance 'sleep $${MTR_DELAY_SECONDS:-1s}', 'MTR_SUITE_LIST=$([ "$MTR_FULL_SUITE" == true ] && echo "' + mtr_full_set + '" || echo "$MTR_SUITE_LIST")', @@ -462,7 +473,7 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise') 'cd mariadb-columnstore-regression-test', 'git rev-parse --abbrev-ref HEAD && git rev-parse HEAD', 'cd ..', - 'docker run --shm-size=500m --env OS=' + result + ' --env PACKAGES_URL=' + packages_url + ' --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --name regression$${DRONE_BUILD_NUMBER} --ulimit core=-1 --privileged --detach ' + img + ' ' + init + ' --unit=basic.target'] + 'docker run --shm-size=500m --memory 10g --env OS=' + result + ' --env PACKAGES_URL=' + packages_url + ' --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --name regression$${DRONE_BUILD_NUMBER} --ulimit core=-1 --privileged --detach ' + img + ' ' + init + ' --unit=basic.target'] + prepareTestStage(dockerImage('regression'), pkg_format, result, true) + [ if (platform == 'centos:7') then @@ -482,11 +493,10 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise') execInnerDocker('sed -i "/^.mariadb.$/a lower_case_table_names=1" ' + config_path_prefix + 'server.cnf', dockerImage('regression')), // set default client character set to utf-8 execInnerDocker('sed -i "/^.client.$/a default-character-set=utf8" ' + config_path_prefix + 'client.cnf',dockerImage('regression')), + // Set RAM consumption limits to avoid RAM contention b/w mtr andregression steps. - //'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "/usr/bin/mcsSetConfig HashJoin TotalUmMemory 5G"', - //'docker exec -t regressin$${DRONE_BUILD_NUMBER} bash -c "/usr/bin/mcsSetConfig DBBC NumBlocksPct 2G"', - //'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "/usr/bin/mcsSetConfig SystemConfig CGroup $(docker ps --filter=name=regression$${DRONE_BUILD_NUMBER} --quiet --no-trunc)"', - // start mariadb and mariadb-columnstore services + execInnerDocker("/usr/bin/mcsSetConfig SystemConfig CGroup just_no_group_use_local", dockerImage("regression")), + execInnerDocker('systemctl start mariadb',dockerImage('regression')), execInnerDocker('systemctl restart mariadb-columnstore',dockerImage('regression')), // delay regression for manual debugging on live instance diff --git a/build/bootstrap_mcs.sh b/build/bootstrap_mcs.sh index a10cb6d52..9709ce5a2 100755 --- a/build/bootstrap_mcs.sh +++ b/build/bootstrap_mcs.sh @@ -48,6 +48,7 @@ optparse.define short=j long=parallel desc="Number of paralles for build" variab optparse.define short=F long=show-build-flags desc="Print CMake flags, while build" variable=PRINT_CMAKE_FLAGS default=false value=true optparse.define short=c long=cloud desc="Enable cloud storage" variable=CLOUD_STORAGE_ENABLED default=false value=true optparse.define short=f long=do-not-freeze-revision desc="Disable revision freezing, or do not set 'update none' for columnstore submodule in MDB repository" variable=DO_NOT_FREEZE_REVISION default=false value=true +optparse.define short=a long=build-path variable=MARIA_BUILD_PATH default=$MDB_SOURCE_PATH/../MariaDBBuild source $( optparse.build ) @@ -99,7 +100,7 @@ select_branch() fi cd - > /dev/null - message "Columnstore will be built from $color_yellow$CURRENT_BRANCH$color_normal branch" + message "Columnstore will be built from $color_yellow$CURRENT_BRANCH$color_cyan branch" } install_deps() @@ -196,6 +197,7 @@ clean_old_installation() rm -rf /var/lib/columnstore/local/ rm -rf /var/lib/columnstore/storagemanager/* rm -rf /var/log/mariadb/columnstore/* + rm -rf /etc/mysql/mariadb.conf.d/columnstore.cnf /etc/my.cnf.d/columnstore.cnf rm -rf /tmp/* rm -rf $REPORT_PATH rm -rf /var/lib/mysql @@ -206,8 +208,11 @@ clean_old_installation() build() { + MARIA_BUILD_PATH=$(realpath $MARIA_BUILD_PATH) message_split - message "Building sources in $color_yellow$MCS_BUILD_TYPE$color_normal mode" + message "Building sources in $color_yellow$MCS_BUILD_TYPE$color_cyan mode" + message "Compiled artifacts will be written to $color_yellow$MARIA_BUILD_PATH$color_cyan" + mkdir -p $MARIA_BUILD_PATH local MDB_CMAKE_FLAGS="-DWITH_SYSTEMD=yes -DPLUGIN_COLUMNSTORE=YES @@ -324,12 +329,12 @@ build() fi message "Configuring cmake silently" - ${CMAKE_BIN_NAME} -DCMAKE_BUILD_TYPE=$MCS_BUILD_TYPE $MDB_CMAKE_FLAGS . | spinner + ${CMAKE_BIN_NAME} -DCMAKE_BUILD_TYPE=$MCS_BUILD_TYPE $MDB_CMAKE_FLAGS -S$MDB_SOURCE_PATH -B$MARIA_BUILD_PATH | spinner message_split - ${CMAKE_BIN_NAME} --build . -j $CPUS | onelinearizator && \ + ${CMAKE_BIN_NAME} --build $MARIA_BUILD_PATH -j $CPUS | onelinearizator && \ message "Installing silently" && - ${CMAKE_BIN_NAME} --install . | spinner 30 + ${CMAKE_BIN_NAME} --install $MARIA_BUILD_PATH | spinner 30 if [ $? -ne 0 ]; then message_split diff --git a/mysql-test/columnstore/basic/disabled.def b/mysql-test/columnstore/basic/disabled.def index 171b471ff..aa0cfc49c 100644 --- a/mysql-test/columnstore/basic/disabled.def +++ b/mysql-test/columnstore/basic/disabled.def @@ -6,3 +6,4 @@ mcs118_charset_negative : mcs error code has changed 2022-07-07 roman.navrotskiy mcs16_functions_define_call_drop : 2022-07-08 roman.navrotskiy@mariadb.com udf_calshowpartitions : unstable values for min/max in the output(sometimes N/A, sometimes numbers) 2022-07-26 roman.nozdrin@mariadb.com mcs3_create_table_charset_collations : 10.6 vs 10.9 show create table difference +pron : pron is not threadsfe and doesn't work with PrimProc TDB: leonid.fedorov@mariadb.com diff --git a/utils/common/cgroupconfigurator.cpp b/utils/common/cgroupconfigurator.cpp index c950c99ae..d01c9e2bb 100644 --- a/utils/common/cgroupconfigurator.cpp +++ b/utils/common/cgroupconfigurator.cpp @@ -79,11 +79,11 @@ CGroupConfigurator::CGroupConfigurator() else cGroupDefined = true; + if (cGroupName == "just_no_group_use_local") + cGroupName = std::string{}; ifstream v2Check("/sys/fs/cgroup/cgroup.controllers"); cGroupVersion_ = (v2Check) ? v2 : v1; - - } CGroupConfigurator::~CGroupConfigurator()