From a0893a57dbf6aa7345bfa59435aa5de77f3482eb Mon Sep 17 00:00:00 2001 From: Andrey Piskunov Date: Tue, 14 Feb 2023 15:11:58 +0200 Subject: [PATCH] Add service logs to CI build (#2730) * Add service logs to CI build * Run logs.sh in drone * write to core dir * Proper dir and better script * Fix typo * load logs.sh in regr --- .drone.jsonnet | 13 +++++++++++++ core_dumps/logs.sh | 22 ++++++++++++++++++++++ 2 files changed, 35 insertions(+) create mode 100755 core_dumps/logs.sh diff --git a/.drone.jsonnet b/.drone.jsonnet index 2b99b11ec..55197b029 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -67,6 +67,7 @@ local core_dump_format = 'https://raw.githubusercontent.com/mariadb-corporation/ local core_dump_check = 'https://raw.githubusercontent.com/mariadb-corporation/mariadb-columnstore-engine/develop/core_dumps/core_dump_check.sh'; local core_dump_drop = 'https://raw.githubusercontent.com/mariadb-corporation/mariadb-columnstore-engine/develop/core_dumps/core_dump_drop.sh'; local ansi2html = 'https://raw.githubusercontent.com/mariadb-corporation/mariadb-columnstore-engine/develop/core_dumps/ansi2html.sh'; +local logs = 'https://raw.githubusercontent.com/mariadb-corporation/mariadb-columnstore-engine/with_service_logs/core_dumps/logs.sh'; local platformMap(platform, arch) = local platform_map = { @@ -227,10 +228,12 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise') 'docker exec -t smoke$${DRONE_BUILD_NUMBER} bash -c "wget ' + core_dump_check + '"', 'docker exec -t smoke$${DRONE_BUILD_NUMBER} bash -c "wget ' + core_dump_drop + '"', 'docker exec -t smoke$${DRONE_BUILD_NUMBER} bash -c "wget ' + ansi2html + '"', + 'docker exec -t smoke$${DRONE_BUILD_NUMBER} bash -c "wget ' + logs + '"', 'docker exec -t smoke$${DRONE_BUILD_NUMBER} bash -c "chmod +x core_dump_format.sh"', 'docker exec -t smoke$${DRONE_BUILD_NUMBER} bash -c "chmod +x core_dump_drop.sh"', 'docker exec -t smoke$${DRONE_BUILD_NUMBER} bash -c "chmod +x core_dump_check.sh"', 'docker exec -t smoke$${DRONE_BUILD_NUMBER} bash -c "chmod +x ansi2html.sh"', + 'docker exec -t smoke$${DRONE_BUILD_NUMBER} bash -c "chmod +x logs.sh"', if (std.split(platform, ':')[0] == 'centos' || std.split(platform, ':')[0] == 'rockylinux') then 'docker exec -t smoke$${DRONE_BUILD_NUMBER} bash -c "yum install -y gdb gawk epel-release which rsyslog hostname procps-ng && yum install -y /' + result + '/*.' + pkg_format + '"' else '', if (pkg_format == 'deb') then 'docker exec -t smoke$${DRONE_BUILD_NUMBER} bash -c "apt update --yes && apt install -y gdb gawk rsyslog hostname && apt install -y -f /' + result + '/*.' + pkg_format + '"', 'sleep $${SMOKE_DELAY_SECONDS:-1s}', @@ -263,10 +266,12 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise') 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "wget ' + core_dump_check + '"', 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "wget ' + core_dump_drop + '"', 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "wget ' + ansi2html + '"', + 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "wget ' + logs + '"', 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "chmod +x core_dump_format.sh"', 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "chmod +x core_dump_check.sh"', 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "chmod +x core_dump_drop.sh"', 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "chmod +x ansi2html.sh"', + 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "chmod +x logs.sh"', if (std.split(platform, ':')[0] == 'centos' || std.split(platform, ':')[0] == 'rockylinux') then 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "yum install -y wget gawk gdb epel-release diffutils which rsyslog hostname patch perl cracklib-dicts procps-ng && yum install -y /' + result + '/*.' + pkg_format + '"' else '', if (pkg_format == 'deb') then 'docker exec -t mtr$${DRONE_BUILD_NUMBER} sed -i "s/exit 101/exit 0/g" /usr/sbin/policy-rc.d', if (pkg_format == 'deb') then 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "apt update --yes && apt install -y wget gawk gdb rsyslog hostname patch && apt install -y -f /' + result + '/*.' + pkg_format + '"' else '', @@ -301,8 +306,10 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise') 'docker exec -t mtr$${DRONE_BUILD_NUMBER} cat /var/log/mariadb/columnstore/debug.log || echo "missing columnstore debug.log"', 'echo "---------- end columnstore debug log ----------"', 'echo "---------- end columnstore debug log ----------"', + 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "/logs.sh mtr"', 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "/core_dump_check.sh core /core/ Mtr"', 'docker cp mtr$${DRONE_BUILD_NUMBER}:/core/ /drone/src/' + result + '/', + 'docker cp mtr$${DRONE_BUILD_NUMBER}:/unit_logs/ /drone/src/' + result + '/', 'docker cp mtr$${DRONE_BUILD_NUMBER}:' + mtr_path + '/var/log /drone/src/' + result + '/mtr-logs || echo "missing ' + mtr_path + '/var/log"', 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "/core_dump_drop.sh core"', 'docker stop mtr$${DRONE_BUILD_NUMBER} && docker rm mtr$${DRONE_BUILD_NUMBER} || echo "cleanup mtr failure"', @@ -339,10 +346,12 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise') 'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "wget ' + core_dump_check + '"', 'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "wget ' + core_dump_drop + '"', 'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "wget ' + ansi2html + '"', + 'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "wget ' + logs + '"', 'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "chmod +x core_dump_format.sh"', 'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "chmod +x core_dump_check.sh"', 'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "chmod +x core_dump_drop.sh"', 'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "chmod +x ansi2html.sh"', + 'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "chmod +x logs.sh"', 'docker cp mariadb-columnstore-regression-test regression$${DRONE_BUILD_NUMBER}:/', // list storage manager binary 'ls -la /mdb/' + builddir + '/storage/columnstore/columnstore/storage-manager', @@ -387,8 +396,10 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise') 'echo "---------- start columnstore debug log ----------"', 'docker exec -t smoke$${DRONE_BUILD_NUMBER} cat /var/log/mariadb/columnstore/debug.log || echo "missing columnstore debug.log"', 'echo "---------- end columnstore debug log ----------"', + 'docker exec -t smoke$${DRONE_BUILD_NUMBER} bash -c "/logs.sh smoke"', 'docker exec -t smoke$${DRONE_BUILD_NUMBER} bash -c "/core_dump_check.sh core /core/ Smoke"', 'docker cp smoke$${DRONE_BUILD_NUMBER}:/core/ /drone/src/' + result + '/', + 'docker cp smoke$${DRONE_BUILD_NUMBER}:/unit_logs/ /drone/src/' + result + '/', 'ls -l /drone/src/' + result, 'docker exec -t smoke$${DRONE_BUILD_NUMBER} bash -c "/core_dump_drop.sh core"', 'docker stop smoke$${DRONE_BUILD_NUMBER} && docker rm smoke$${DRONE_BUILD_NUMBER} || echo "cleanup smoke failure"', @@ -410,7 +421,9 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise') 'docker cp regression$${DRONE_BUILD_NUMBER}:/mariadb-columnstore-regression-test/mysql/queries/nightly/alltest/testErrorLogs.tgz /drone/src/' + result + '/ || echo "missing testErrorLogs.tgz"', 'docker exec -t --workdir /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest regression$${DRONE_BUILD_NUMBER} bash -c "tar czf testErrorLogs2.tgz *.log /var/log/mariadb/columnstore" || echo "failed to grab regression results"', 'docker cp regression$${DRONE_BUILD_NUMBER}:/mariadb-columnstore-regression-test/mysql/queries/nightly/alltest/testErrorLogs2.tgz /drone/src/' + result + '/ || echo "missing testErrorLogs.tgz"', + 'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "/logs.sh regression"', 'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "/core_dump_check.sh core /core/ Regression"', + 'docker cp regression$${DRONE_BUILD_NUMBER}:/unit_logs/ /drone/src/' + result + '/', 'docker cp regression$${DRONE_BUILD_NUMBER}:/core/ /drone/src/' + result + '/', 'ls -l /drone/src/' + result, 'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "/core_dump_drop.sh core"', diff --git a/core_dumps/logs.sh b/core_dumps/logs.sh new file mode 100755 index 000000000..cea6fa92d --- /dev/null +++ b/core_dumps/logs.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env sh + +set -x + +SCRIPT_LOCATION=$(dirname "$0") +STEP_NAME=$1 +DIR_NAME="unit_logs" + +mkdir "$DIR_NAME" + +dump_log () +{ + name=$1 + journalctl -u "$name".service > "$DIR_NAME"/"${name}_${STEP_NAME}".log +} + +dump_log "mcs-ddlproc" +dump_log "mcs-dmlproc" +dump_log "mcs-loadbrm" +dump_log "mcs-primproc" +dump_log "mcs-workernode@1" +dump_log "mcs-writeengineserver"