MCOL-5478 upgrade tests (#2815)

* Squash all * Typos * Only load shared on centos * Debug * Install rsyslog * More exceptions * One more exception * rename pkgs * Remove unnecessary copying * deb doesn't work on arm with 1-4 * Disable CentOS * Remove flights * Use setup-repo * Exclude deb12
2025-04-18 21:44:02 +03:00 · 2023-08-04 18:43:52 +03:00 · 2023-08-04 18:43:52 +03:00 · f2affca162
commit f2affca162
parent 896e8dd769
7 changed files with 256 additions and 2 deletions
--- a/.drone.jsonnet
+++ b/.drone.jsonnet
@ -122,9 +122,9 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise')
  local brancht = if (branch == '**') then '' else branch + '-',
  local result = std.strReplace(std.strReplace(platform, ':', ''), '/', '-'),

-  local publish_pkg_url = 'https://cspkg.s3.amazonaws.com/index.html?prefix=' + branchp + event + '/${DRONE_BUILD_NUMBER}/' + server + '/' + arch + '/' + result + '/',
-
  local packages_url = 'https://cspkg.s3.amazonaws.com/' + branchp + event + '/${DRONE_BUILD_NUMBER}/' + server,
+  local publish_pkg_url = "https://cspkg.s3.amazonaws.com/index.html?prefix=" + branchp + event + "/${DRONE_BUILD_NUMBER}/" + server + "/" + arch + "/" + result + "/",
+  local repo_pkg_url_no_res = "https://cspkg.s3.amazonaws.com/" + branchp + event + "/${DRONE_BUILD_NUMBER}/" + server + "/" + arch + "/",

  local container_tags = if (event == 'cron') then [brancht + std.strReplace(event, '_', '-') + '${DRONE_BUILD_NUMBER}', brancht] else [brancht + std.strReplace(event, '_', '-') + '${DRONE_BUILD_NUMBER}'],
  local container_version = branchp + event + '/${DRONE_BUILD_NUMBER}/' + server + '/' + arch,
@ -190,6 +190,8 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise')
    'test001.sh',
  ],

+  local mdb_server_versions = ['10.6.4-1', '10.6.5-2', '10.6.7-3', '10.6.8-4', '10.6.9-5', '10.6.11-6', '10.6.12-7'],
+
  local indexes(arr) = std.range(0, std.length(arr) - 1),

  regression(name, depends_on):: {
@ -251,6 +253,36 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise')
      'docker exec -t smoke$${DRONE_BUILD_NUMBER} mariadb -e "insert into test.t1 values (2); select * from test.t1"',
    ],
  },
+  upgrade(version):: {
+    name: 'upgrade-test from ' + version,
+    depends_on: ['smoke'],
+    image: 'docker',
+    volumes: [pipeline._volumes.docker],
+    environment: {
+      UPGRADE_TOKEN: {
+        from_secret: 'es_token',
+      },
+    },
+    commands: [
+      'docker run --volume /sys/fs/cgroup:/sys/fs/cgroup:ro --env OS=' + result + ' --env PACKAGES_URL=' + packages_url + ' --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --name upgrade$${DRONE_BUILD_NUMBER}' + version + ' --ulimit core=-1 --privileged --detach ' + img + ' ' + init + ' --unit=basic.target',
+      'docker cp core_dumps/. upgrade$${DRONE_BUILD_NUMBER}' + version + ':/',
+      'docker cp setup-repo.sh upgrade$${DRONE_BUILD_NUMBER}' + version + ':/',
+      if (pkg_format == 'deb' && !(result == "debian12") && !(arch == "arm64" && (version == "10.6.4-1" || version == "10.6.5-2" || version == "10.6.7-3" || version == "10.6.8-4")) && !((version == "10.6.4-1" || version == "10.6.8-4") && (result == "debian11" || result == "ubuntu22.04")) && !(result == "ubuntu22.04" && (version == "10.6.5-2" || version == "10.6.7-3"))) then 'docker exec -t upgrade$${DRONE_BUILD_NUMBER}' + version + ' bash -c "./upgrade_setup_deb.sh '+ version + ' ' + result + ' ' + arch + ' ' + repo_pkg_url_no_res +' $${UPGRADE_TOKEN}"',
+      if (std.split(platform, ':')[0] == 'rockylinux' && !(result == "rockylinux9" && (version == "10.6.4-1" || version == "10.6.5-2" || version == "10.6.7-3" || version == "10.6.8-4")) && !(result == "rockylinux8" && arch == 'arm64' && (version == "10.6.5-2" || version == "10.6.7-3" || version == "10.6.8-4"))) then 'docker exec -t upgrade$${DRONE_BUILD_NUMBER}' + version + ' bash -c "./upgrade_setup_rpm.sh '+ version + ' ' + result + ' ' + arch + ' ' + repo_pkg_url_no_res + ' $${UPGRADE_TOKEN}"',
+    ],
+  },
+  upgradelog:: {
+    name: 'upgradelog',
+    depends_on: ['upgrade-test from ' + mdb_server_versions[std.length(mdb_server_versions) - 1]],
+    image: 'docker',
+    volumes: [pipeline._volumes.docker],
+    commands: [
+      'echo',
+    ] + std.map(function(ver) 'docker stop upgrade$${DRONE_BUILD_NUMBER}' + ver + ' && docker rm upgrade$${DRONE_BUILD_NUMBER}' + ver + ' || echo "cleanup upgrade from version ' + ver + ' failure"', mdb_server_versions),
+    when: {
+      status: ['success', 'failure'],
+    },
+  },
  mtr:: {
    name: 'mtr',
    depends_on: ['smoke'],
@ -642,6 +674,8 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise')
               'git config cmake.update-submodules no',
               'rm -rf storage/columnstore/columnstore',
               'cp -r /drone/src /mdb/' + builddir + '/storage/columnstore/columnstore',
+               if (std.split(platform, ':')[0] == 'centos') then 'wget -P /mdb/ https://cspkg.s3.amazonaws.com/MariaDB-Compat/MariaDB-shared-10.1-kvm-rpm-centos74-amd64.rpm',
+               if (std.split(platform, ':')[0] == 'centos') then 'wget -P /mdb/ https://cspkg.s3.amazonaws.com/MariaDB-Compat/MariaDB-shared-5.3-amd64.rpm',
             ],
           },
           {
@ -668,6 +702,7 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise')
             },
             commands: [
               'cd /mdb/' + builddir,
+               'ls -la ../',
               'mkdir ' + result,
               "sed -i 's|.*-d storage/columnstore.*|elif [[ -d storage/columnstore/columnstore/debian ]]|' debian/autobake-deb.sh",
               if (std.startsWith(server, '10.6')) then "sed -i 's/mariadb-server/mariadb-server-10.6/' storage/columnstore/columnstore/debian/control",
@ -751,6 +786,8 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise')
         [pipeline.publish('smokelog')] +
         [pipeline.cmapitest] +
         [pipeline.cmapilog] +
+         [pipeline.upgrade(mdb_server_versions[i]) for i in indexes(mdb_server_versions)] +
+         [pipeline.upgradelog] +
         (if (platform == 'rockylinux:8' && arch == 'amd64') then [pipeline.dockerfile] + [pipeline.dockerhub] + [pipeline.multi_node_mtr] else [pipeline.mtr] + [pipeline.publish('mtr')] + [pipeline.mtrlog] + [pipeline.publish('mtrlog')]) +
         (if (event == 'cron' && platform == 'rockylinux:8' && arch == 'amd64') then [pipeline.publish('mtr latest', 'latest')] else []) +
         [pipeline.prepare_regression] +
--- a/core_dumps/columnstore_schema.sql
+++ b/core_dumps/columnstore_schema.sql
@ -0,0 +1,55 @@
+DROP DATABASE IF EXISTS `columnstore_bts`;
+
+CREATE DATABASE `columnstore_bts`;
+
+USE `columnstore_bts`;
+
+CREATE TABLE `airlines` (
+  `iata_code` varchar(2) DEFAULT NULL,
+  `airline` varchar(30) DEFAULT NULL
+) ENGINE=Columnstore DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_general_ci;
+
+CREATE TABLE `airports` (
+  `iata_code` varchar(3) DEFAULT NULL,
+  `airport` varchar(80) DEFAULT NULL,
+  `city` varchar(30) DEFAULT NULL,
+  `state` varchar(2) DEFAULT NULL,
+  `country` varchar(30) DEFAULT NULL,
+  `latitude` decimal(11,4) DEFAULT NULL,
+  `longitude` decimal(11,4) DEFAULT NULL
+) ENGINE=Columnstore DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_general_ci;
+
+CREATE TABLE `flights` (
+  `year` smallint(6) DEFAULT NULL,
+  `month` tinyint(4) DEFAULT NULL,
+  `day` tinyint(4) DEFAULT NULL,
+  `day_of_week` tinyint(4) DEFAULT NULL,
+  `fl_date` date DEFAULT NULL,
+  `carrier` varchar(2) DEFAULT NULL,
+  `tail_num` varchar(6) DEFAULT NULL,
+  `fl_num` smallint(6) DEFAULT NULL,
+  `origin` varchar(5) DEFAULT NULL,
+  `dest` varchar(5) DEFAULT NULL,
+  `crs_dep_time` varchar(4) DEFAULT NULL,
+  `dep_time` varchar(4) DEFAULT NULL,
+  `dep_delay` decimal(13,2) DEFAULT NULL,
+  `taxi_out` decimal(13,2) DEFAULT NULL,
+  `wheels_off` varchar(4) DEFAULT NULL,
+  `wheels_on` varchar(4) DEFAULT NULL,
+  `taxi_in` decimal(13,2) DEFAULT NULL,
+  `crs_arr_time` varchar(4) DEFAULT NULL,
+  `arr_time` varchar(4) DEFAULT NULL,
+  `arr_delay` decimal(13,2) DEFAULT NULL,
+  `cancelled` decimal(13,2) DEFAULT NULL,
+  `cancellation_code` varchar(20) DEFAULT NULL,
+  `diverted` decimal(13,2) DEFAULT NULL,
+  `crs_elapsed_time` decimal(13,2) DEFAULT NULL,
+  `actual_elapsed_time` decimal(13,2) DEFAULT NULL,
+  `air_time` decimal(13,2) DEFAULT NULL,
+  `distance` decimal(13,2) DEFAULT NULL,
+  `carrier_delay` decimal(13,2) DEFAULT NULL,
+  `weather_delay` decimal(13,2) DEFAULT NULL,
+  `nas_delay` decimal(13,2) DEFAULT NULL,
+  `security_delay` decimal(13,2) DEFAULT NULL,
+  `late_aircraft_delay` decimal(13,2) DEFAULT NULL
+) ENGINE=Columnstore DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_general_ci;
--- a/core_dumps/upgrade_data.sh
+++ b/core_dumps/upgrade_data.sh
@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+
+set -xeuo pipefail
+
+MARIADB=$(which mysql)
+CPIMPORT=$(which cpimport)
+SCHEMA_DIR=$(dirname "$0")
+NAME1='airports'
+NAME2='airlines'
+
+
+get_data ()
+{
+  NAME=$1
+  if curl -o "${SCHEMA_DIR}/${NAME}.csv" -# "https://sample-columnstore-data.s3.us-west-2.amazonaws.com/${NAME}.csv"; then
+     echo -e "Downloaded '${NAME}.csv' ... done\n"
+  else
+     echo -e "Downloading '${NAME}.csv' ... failed"
+     exit 1
+  fi
+}
+
+import_data ()
+{
+    NAME=$1
+    echo -e "\nLoading '${NAME}.csv' with cpimport ..."
+    if ! $CPIMPORT -m 1 -s ',' -E '"' columnstore_bts "${NAME}" -l "${SCHEMA_DIR}/${NAME}.csv"; then
+        echo -e "loading '${NAME}.csv' ... fail"
+        exit 1
+    fi
+}
+
+
+if $MARIADB <"${SCHEMA_DIR}"/columnstore_schema.sql &>/dev/null; then
+    echo -e "Creating 'columnstore_bts' schema..." "done"
+else
+    echo -e "Creating 'columnstore_bts' schema..." "fail"
+    exit 1
+fi
+
+get_data "$NAME1"
+get_data "$NAME2"
+
+import_data "$NAME1"
+import_data "$NAME2"
--- a/core_dumps/upgrade_setup_cent.sh
+++ b/core_dumps/upgrade_setup_cent.sh
@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+set -xeuo pipefail
+
+VERSION="$1"
+RESULT="$2"
+ARCH="$3"
+LINK="$4"
+UPGRADE_TOKEN="$5"
+
+yum install -y wget which procps-ng diffutils
+wget https://dlm.mariadb.com/enterprise-release-helpers/mariadb_es_repo_setup -O mariadb_es_repo_setup
+chmod +x mariadb_es_repo_setup
+bash -c "./mariadb_es_repo_setup --token=${UPGRADE_TOKEN} --apply --mariadb-server-version=${VERSION} --skip-maxscale --skip-tools --skip-check-installed"
+yum -y install MariaDB-server MariaDB-client MariaDB-columnstore-engine MariaDB-columnstore-engine-debuginfo
+
+systemctl start mariadb
+systemctl start mariadb-columnstore
+bash -c "./upgrade_data.sh"
+bash -c "./upgrade_verify.sh"
+
+cd /etc/yum.repos.d/
+touch repo.repo
+cat <<EOF > repo.repo
+[repo]
+name = repo
+baseurl = ${LINK}${RESULT}/
+enabled = 1
+gpgcheck = 0
+module_hotfixes=1
+EOF
+
+cd /
+
+yum -y update MariaDB-server MariaDB-client MariaDB-columnstore-engine MariaDB-columnstore-engine-debuginfo
+bash -c "./upgrade_verify.sh"
--- a/core_dumps/upgrade_setup_deb.sh
+++ b/core_dumps/upgrade_setup_deb.sh
@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+
+set -xeuo pipefail
+
+VERSION="$1"
+RESULT="$2"
+ARCH="$3"
+LINK="$4"
+UPGRADE_TOKEN="$5"
+
+apt install --yes rsyslog
+sed -i "s/exit 101/exit 0/g" /usr/sbin/policy-rc.d
+bash -c "apt update --yes && apt install -y procps wget curl"
+wget https://dlm.mariadb.com/enterprise-release-helpers/mariadb_es_repo_setup -O mariadb_es_repo_setup
+chmod +x mariadb_es_repo_setup
+bash -c "./mariadb_es_repo_setup --token=${UPGRADE_TOKEN} --apply --mariadb-server-version=${VERSION} --skip-maxscale --skip-tools"
+apt update --yes
+apt install --yes mariadb-server mariadb-client mariadb-plugin-columnstore
+systemctl start mariadb
+systemctl start mariadb-columnstore
+bash -c "./upgrade_data.sh"
+bash -c "./upgrade_verify.sh"
+
+touch /etc/apt/auth.conf
+cat << EOF > /etc/apt/auth.conf
+machine ${LINK}${RESULT}/
+EOF
+
+bash -c "./setup-repo.sh"
+
+apt upgrade --yes
+bash -c "./upgrade_verify.sh"
--- a/core_dumps/upgrade_setup_rpm.sh
+++ b/core_dumps/upgrade_setup_rpm.sh
@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+set -xeuo pipefail
+
+VERSION="$1"
+RESULT="$2"
+ARCH="$3"
+LINK="$4"
+UPGRADE_TOKEN="$5"
+
+yum clean all
+rm -rf /var/cache/dnf/*
+yum install -y wget which procps-ng diffutils rsyslog
+wget https://dlm.mariadb.com/enterprise-release-helpers/mariadb_es_repo_setup -O mariadb_es_repo_setup
+chmod +x mariadb_es_repo_setup
+bash -c "./mariadb_es_repo_setup --token=${UPGRADE_TOKEN} --apply --mariadb-server-version=${VERSION} --skip-maxscale --skip-tools --skip-check-installed"
+dnf repo-pkgs mariadb-es-main list
+dnf -y install MariaDB-server MariaDB-client MariaDB-columnstore-engine MariaDB-columnstore-engine-debuginfo
+
+systemctl start mariadb
+systemctl start mariadb-columnstore
+bash -c "./upgrade_data.sh"
+bash -c "./upgrade_verify.sh"
+
+bash -c "./setup-repo.sh"
+
+dnf repo-pkgs repo list
+dnf -y update MariaDB-server MariaDB-client MariaDB-columnstore-engine MariaDB-columnstore-engine-debuginfo
+bash -c "./upgrade_verify.sh"
--- a/core_dumps/upgrade_verify.sh
+++ b/core_dumps/upgrade_verify.sh
@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+set -xeuo pipefail
+
+SCHEMA_DIR=$(dirname "$0")
+NAME1='airports'
+NAME2='airlines'
+
+
+test_data ()
+{
+    NAME=$1
+    mariadb --init-command="SET sql_mode=''" -vvv -e "select * into outfile '/tmp/${NAME}.test.csv' FIELDS TERMINATED BY ',' LINES TERMINATED BY '\n' FROM columnstore_bts.${NAME};"
+    diff <(tail -n +2 "${NAME}.csv") <(tail -n +2  "/tmp/${NAME}.test.csv")
+    rm "/tmp/${NAME}.test.csv"
+}
+
+
+test_data "$NAME1"
+test_data "$NAME2"