fix(cgroups)!: Containers memory limits for CI (#3108) (#3209)

Limit test containers by memory, fix cgroup path inside the containers by introducing new ugly setting name --------- Co-authored-by: drrtuy <roman.nozdrin@mariadb.com> Co-authored-by: Roman Nozdrin <rnozdrin@mariadb.com>
2025-07-29 08:21:15 +03:00 · 2024-06-20 13:48:20 +04:00
parent e6c02c8832
commit cce0f6ab0c
4 changed files with 36 additions and 20 deletions
--- a/.drone.jsonnet
+++ b/.drone.jsonnet
@ -251,6 +251,18 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise')
    if (pkg_format == 'deb')
      then execInnerDocker('sed -i "s/exit 101/exit 0/g" /usr/sbin/policy-rc.d', dockerImage),

+    'echo "Docker CGroups opts here"',
+    'ls -al /sys/fs/cgroup/cgroup.controllers || true ',
+    'ls -al /sys/fs/cgroup/ || true ',
+    'ls -al /sys/fs/cgroup/memory || true',
+    "docker ps --filter=name=" + dockerImage,
+
+    execInnerDocker('echo "Inner Docker CGroups opts here"', dockerImage),
+    execInnerDocker('ls -al /sys/fs/cgroup/cgroup.controllers || true', dockerImage),
+    execInnerDocker('ls -al /sys/fs/cgroup/ || true', dockerImage),
+    execInnerDocker('ls -al /sys/fs/cgroup/memory || true', dockerImage),
+
+
    execInnerDocker('mkdir core', dockerImage),
    execInnerDocker('chmod 777 core', dockerImage),
    'docker cp core_dumps/. ' + dockerImage  +  ':/',
@ -312,13 +324,14 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise')
    image: 'docker',
    volumes: [pipeline._volumes.docker],
    commands: [
-      'docker run --env OS=' + result + ' --env PACKAGES_URL=' + packages_url + ' --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --name smoke$${DRONE_BUILD_NUMBER} --ulimit core=-1 --privileged --detach ' + img + ' ' + init + ' --unit=basic.target']
+      'docker run --memory 3g --env OS=' + result + ' --env PACKAGES_URL=' + packages_url + ' --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --name smoke$${DRONE_BUILD_NUMBER} --ulimit core=-1 --privileged --detach ' + img + ' ' + init + ' --unit=basic.target']
      + prepareTestStage(dockerImage("smoke"), pkg_format, result, true) + [
      installEngine(dockerImage("smoke"), pkg_format),
      'sleep $${SMOKE_DELAY_SECONDS:-1s}',
      // start mariadb and mariadb-columnstore services and run simple query
      execInnerDocker('systemctl start mariadb', dockerImage("smoke")),
-      execInnerDocker('systemctl start mariadb-columnstore', dockerImage("smoke")),
+      execInnerDocker("/usr/bin/mcsSetConfig SystemConfig CGroup just_no_group_use_local", dockerImage("smoke")),
+      execInnerDocker('systemctl restart mariadb-columnstore', dockerImage("smoke")),
      execInnerDocker('mariadb -e "create database if not exists test; create table test.t1 (a int) engine=Columnstore; insert into test.t1 values (1); select * from test.t1"',
                      dockerImage("smoke")),

@ -340,6 +353,7 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise')
      },
    },
    commands: [
+      // why do we mount cgroups here, but miss it on other steps?
      'docker run --volume /sys/fs/cgroup:/sys/fs/cgroup:ro --env OS=' + result + ' --env PACKAGES_URL=' + packages_url + ' --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --env UCF_FORCE_CONFNEW=1 --name upgrade$${DRONE_BUILD_NUMBER}' + version + ' --ulimit core=-1 --privileged --detach ' + img + ' ' + init + ' --unit=basic.target']
      + prepareTestStage(dockerImage('upgrade') + version, pkg_format, result, false) + [
      if (pkg_format == 'deb')
@ -375,7 +389,7 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise')
      MTR_FULL_SUITE: '${MTR_FULL_SUITE:-false}',
    },
    commands: [
-      'docker run --shm-size=500m --env MYSQL_TEST_DIR=' + mtr_path + ' --env OS=' + result + ' --env PACKAGES_URL=' + packages_url + ' --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --name mtr$${DRONE_BUILD_NUMBER} --ulimit core=-1 --privileged --detach ' + img + ' ' + init + ' --unit=basic.target']
+      'docker run --shm-size=500m --memory 8g --env MYSQL_TEST_DIR=' + mtr_path + ' --env OS=' + result + ' --env PACKAGES_URL=' + packages_url + ' --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --name mtr$${DRONE_BUILD_NUMBER} --ulimit core=-1 --privileged --detach ' + img + ' ' + init + ' --unit=basic.target']
      + prepareTestStage('mtr$${DRONE_BUILD_NUMBER}', pkg_format, result, true) + [
      installEngine(dockerImage("mtr"), pkg_format),
      'docker cp mysql-test/columnstore mtr$${DRONE_BUILD_NUMBER}:' + mtr_path + '/suite/',
@ -384,14 +398,11 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise')
      execInnerDocker("bash -c 'sed -i /ProtectSystem/d $(systemctl show --property FragmentPath mariadb | sed s/FragmentPath=//)'", dockerImage('mtr')),
      execInnerDocker('systemctl daemon-reload', dockerImage("mtr")),
      execInnerDocker('systemctl start mariadb', dockerImage("mtr")),
+      // Set RAM consumption limits to avoid RAM contention b/w mtr and regression steps.
+      execInnerDocker("/usr/bin/mcsSetConfig SystemConfig CGroup just_no_group_use_local", dockerImage("mtr")),
      execInnerDocker('mariadb -e "create database if not exists test;"', dockerImage("mtr")),
      execInnerDocker('systemctl restart mariadb-columnstore', dockerImage("mtr")),

-      // Set RAM consumption limits to avoid RAM contention b/w mtr and regression steps.
-      //'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "/usr/bin/mcsSetConfig HashJoin TotalUmMemory 4G"',
-      //'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "/usr/bin/mcsSetConfig DBBC NumBlocksPct 1G"',
-      //'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "/usr/bin/mcsSetConfig SystemConfig CGroup $(docker ps --filter=name=mtr$${DRONE_BUILD_NUMBER} --quiet --no-trunc)"',
-
      // delay mtr for manual debugging on live instance
      'sleep $${MTR_DELAY_SECONDS:-1s}',
      'MTR_SUITE_LIST=$([ "$MTR_FULL_SUITE" == true ] && echo "' + mtr_full_set + '" || echo "$MTR_SUITE_LIST")',
@ -462,7 +473,7 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise')
      'cd mariadb-columnstore-regression-test',
      'git rev-parse --abbrev-ref HEAD && git rev-parse HEAD',
      'cd ..',
-      'docker run --shm-size=500m --env OS=' + result + ' --env PACKAGES_URL=' + packages_url + ' --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --name regression$${DRONE_BUILD_NUMBER} --ulimit core=-1 --privileged --detach ' + img + ' ' + init + ' --unit=basic.target']
+      'docker run --shm-size=500m --memory 10g --env OS=' + result + ' --env PACKAGES_URL=' + packages_url + ' --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --name regression$${DRONE_BUILD_NUMBER} --ulimit core=-1 --privileged --detach ' + img + ' ' + init + ' --unit=basic.target']
      + prepareTestStage(dockerImage('regression'), pkg_format, result, true) + [

      if (platform == 'centos:7') then
@ -482,11 +493,10 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise')
      execInnerDocker('sed -i "/^.mariadb.$/a lower_case_table_names=1" ' + config_path_prefix + 'server.cnf', dockerImage('regression')),
      // set default client character set to utf-8
      execInnerDocker('sed -i "/^.client.$/a default-character-set=utf8" ' + config_path_prefix + 'client.cnf',dockerImage('regression')),
+
      // Set RAM consumption limits to avoid RAM contention b/w mtr andregression steps.
-      //'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "/usr/bin/mcsSetConfig HashJoin TotalUmMemory 5G"',
-      //'docker exec -t regressin$${DRONE_BUILD_NUMBER} bash -c "/usr/bin/mcsSetConfig DBBC NumBlocksPct 2G"',
-      //'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "/usr/bin/mcsSetConfig SystemConfig CGroup $(docker ps --filter=name=regression$${DRONE_BUILD_NUMBER} --quiet --no-trunc)"',
-      // start mariadb and mariadb-columnstore services
+      execInnerDocker("/usr/bin/mcsSetConfig SystemConfig CGroup just_no_group_use_local", dockerImage("regression")),
+
      execInnerDocker('systemctl start mariadb',dockerImage('regression')),
      execInnerDocker('systemctl restart mariadb-columnstore',dockerImage('regression')),
      // delay regression for manual debugging on live instance
--- a/build/bootstrap_mcs.sh
+++ b/build/bootstrap_mcs.sh
@ -48,6 +48,7 @@ optparse.define short=j long=parallel desc="Number of paralles for build" variab
 optparse.define short=F long=show-build-flags desc="Print CMake flags, while build" variable=PRINT_CMAKE_FLAGS default=false value=true
 optparse.define short=c long=cloud desc="Enable cloud storage" variable=CLOUD_STORAGE_ENABLED default=false value=true
 optparse.define short=f long=do-not-freeze-revision desc="Disable revision freezing, or do not set 'update none' for columnstore submodule in MDB repository" variable=DO_NOT_FREEZE_REVISION default=false value=true
+optparse.define short=a long=build-path variable=MARIA_BUILD_PATH default=$MDB_SOURCE_PATH/../MariaDBBuild

 source $( optparse.build )

@ -99,7 +100,7 @@ select_branch()
    fi

    cd - > /dev/null
-    message "Columnstore will be built from $color_yellow$CURRENT_BRANCH$color_normal branch"
+    message "Columnstore will be built from $color_yellow$CURRENT_BRANCH$color_cyan branch"
 }

 install_deps()
@ -196,6 +197,7 @@ clean_old_installation()
    rm -rf /var/lib/columnstore/local/
    rm -rf /var/lib/columnstore/storagemanager/*
    rm -rf /var/log/mariadb/columnstore/*
+    rm -rf /etc/mysql/mariadb.conf.d/columnstore.cnf /etc/my.cnf.d/columnstore.cnf
    rm -rf /tmp/*
    rm -rf $REPORT_PATH
    rm -rf /var/lib/mysql
@ -206,8 +208,11 @@ clean_old_installation()

 build()
 {
+    MARIA_BUILD_PATH=$(realpath $MARIA_BUILD_PATH)
    message_split
-    message "Building sources in $color_yellow$MCS_BUILD_TYPE$color_normal mode"
+    message "Building sources in $color_yellow$MCS_BUILD_TYPE$color_cyan mode"
+    message "Compiled artifacts will be written to $color_yellow$MARIA_BUILD_PATH$color_cyan"
+    mkdir -p $MARIA_BUILD_PATH

    local MDB_CMAKE_FLAGS="-DWITH_SYSTEMD=yes
                     -DPLUGIN_COLUMNSTORE=YES
@ -324,12 +329,12 @@ build()
    fi

    message "Configuring cmake silently"
-    ${CMAKE_BIN_NAME} -DCMAKE_BUILD_TYPE=$MCS_BUILD_TYPE $MDB_CMAKE_FLAGS . | spinner
+    ${CMAKE_BIN_NAME} -DCMAKE_BUILD_TYPE=$MCS_BUILD_TYPE $MDB_CMAKE_FLAGS -S$MDB_SOURCE_PATH -B$MARIA_BUILD_PATH | spinner
    message_split

-    ${CMAKE_BIN_NAME} --build . -j $CPUS | onelinearizator && \
+    ${CMAKE_BIN_NAME} --build $MARIA_BUILD_PATH -j $CPUS | onelinearizator && \
    message "Installing silently" &&
-    ${CMAKE_BIN_NAME} --install . | spinner 30
+    ${CMAKE_BIN_NAME} --install $MARIA_BUILD_PATH | spinner 30

    if [ $? -ne 0 ]; then
        message_split
--- a/mysql-test/columnstore/basic/disabled.def
+++ b/mysql-test/columnstore/basic/disabled.def
@ -6,3 +6,4 @@ mcs118_charset_negative : mcs error code has changed 2022-07-07 roman.navrotskiy
 mcs16_functions_define_call_drop : 2022-07-08 roman.navrotskiy@mariadb.com
 udf_calshowpartitions : unstable values for min/max in the output(sometimes N/A, sometimes numbers) 2022-07-26 roman.nozdrin@mariadb.com
 mcs3_create_table_charset_collations : 10.6 vs 10.9 show create table difference
+pron : pron is not threadsfe and doesn't work with PrimProc TDB: leonid.fedorov@mariadb.com
--- a/utils/common/cgroupconfigurator.cpp
+++ b/utils/common/cgroupconfigurator.cpp
@ -79,11 +79,11 @@ CGroupConfigurator::CGroupConfigurator()
  else
    cGroupDefined = true;

+  if (cGroupName == "just_no_group_use_local")
+    cGroupName = std::string{};

  ifstream v2Check("/sys/fs/cgroup/cgroup.controllers");
  cGroupVersion_ = (v2Check) ? v2 : v1;
-
-
 }

 CGroupConfigurator::~CGroupConfigurator()