database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-10-31 18:30:33 +03:00
Code Activity

Merge branch 'stable-23.10' into feat/MCOL-6072-parallel-scan-4-CES-4

Browse Source
This commit is contained in:
drrtuy
2025-09-04 15:47:40 +00:00
parent 8c0c85080c 77c3aea6eb
commit ed2c89196b
49 changed files with 1371 additions and 534 deletions
Download Patch File Download Diff File Expand all files Collapse all files

104
.drone.jsonnet
View File

@@ -7,6 +7,11 @@ local servers = {
[current_branch]: ["10.6-enterprise"],
};
local extra_servers = {
[current_branch]: ["11.4-enterprise"],
};
local platforms = {
[current_branch]: ["rockylinux:8", "rockylinux:9", "rockylinux:10", "debian:12", "ubuntu:22.04", "ubuntu:24.04"],
};
@@ -19,11 +24,12 @@ local builddir = "verylongdirnameforverystrangecpackbehavior";
local get_build_command(command) = "bash /mdb/" + builddir + "/storage/columnstore/columnstore/build/" + command + " ";
local clang(version) = [get_build_command("install_clang_deb.sh " + version),
local clang(version) = [
get_build_command("install_clang_deb.sh " + version),
get_build_command("update-clang-version.sh " + version + " 100"),
get_build_command("install_libc++.sh " + version),
"export CC=/usr/bin/clang",
"export CXX=/usr/bin/clang++"
"export CXX=/usr/bin/clang++",
];
local customEnvCommandsMap = {
@@ -36,9 +42,9 @@ local customEnvCommands(envkey, builddir) =
local customBootstrapParamsForExisitingPipelines(envkey) =
# errorprone if we pass --custom-cmake-flags twice, the last one will win
// errorprone if we pass --custom-cmake-flags twice, the last one will win
local customBootstrapMap = {
"ubuntu:24.04": "--custom-cmake-flags '-DCOLUMNSTORE_ASAN_FOR_UNITTESTS=YES'",
//'ubuntu:24.04': "--custom-cmake-flags '-DCOLUMNSTORE_ASAN_FOR_UNITTESTS=YES'",
};
(if (std.objectHas(customBootstrapMap, envkey))
then customBootstrapMap[envkey] else "");
@@ -48,8 +54,8 @@ local customBootstrapParamsForAdditionalPipelinesMap = {
TSAN: "--tsan",
UBSan: "--ubsan",
MSan: "--msan",
"libcpp": "--libcpp --skip-unit-tests",
"gcc-toolset": "--gcc-toolset-for-rocky-8"
libcpp: "--libcpp --skip-unit-tests",
"gcc-toolset": "--gcc-toolset-for-rocky-8",
};
local customBuildFlags(buildKey) =
@@ -92,8 +98,10 @@ local upgrade_test_lists = {
};
local make_clickable_link(link) = "echo -e '\\e]8;;" + link + "\\e\\\\" + link + "\\e]8;;\\e\\\\'";
local echo_running_on = ["echo running on ${DRONE_STAGE_MACHINE}",
make_clickable_link("https://us-east-1.console.aws.amazon.com/ec2/home?region=us-east-1#Instances:search=:${DRONE_STAGE_MACHINE};v=3;$case=tags:true%5C,client:false;$regex=tags:false%5C,client:false;sort=desc:launchTime")];
local echo_running_on = [
"echo running on ${DRONE_STAGE_MACHINE}",
make_clickable_link("https://us-east-1.console.aws.amazon.com/ec2/home?region=us-east-1#Instances:search=:${DRONE_STAGE_MACHINE};v=3;$case=tags:true%5C,client:false;$regex=tags:false%5C,client:false;sort=desc:launchTime"),
];
local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", customBootstrapParamsKey="", customBuildEnvCommandsMapKey="", ignoreFailureStepList=[]) = {
local pkg_format = if (std.split(platform, ":")[0] == "rockylinux") then "rpm" else "deb",
@@ -145,13 +153,13 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
"CURRENT_VERSION=${COLUMNSTORE_VERSION_MAJOR}.${COLUMNSTORE_VERSION_MINOR}.${COLUMNSTORE_VERSION_PATCH} && " +
"aws s3 rm s3://cspkg/" + branchp + eventp + "/" + server + "/" + arch + "/" + result + "/ " +
"--recursive " +
"--exclude \"*\" " +
'--exclude "*" ' +
// include only debs/rpms with columnstore in names
"--include \"*columnstore*.deb\" " +
"--include \"*columnstore*.rpm\" " +
'--include "*columnstore*.deb" ' +
'--include "*columnstore*.rpm" ' +
// but do not delete the ones matching CURRENT_VERSION
"--exclude \"*${CURRENT_VERSION}*.deb\" " +
"--exclude \"*${CURRENT_VERSION}*.rpm\" " +
'--exclude "*${CURRENT_VERSION}*.deb" ' +
'--exclude "*${CURRENT_VERSION}*.rpm" ' +
"--only-show-errors",
"aws s3 sync " + result + "/" + " s3://cspkg/" + branchp + eventp + "/" + server + "/" + arch + "/" + result + " --only-show-errors",
@@ -212,9 +220,9 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
local reportTestStage(containerName, result, stage) =
'sh -c "apk add bash && ' + get_build_command("report_test_stage.sh") +
' --container-name ' + containerName +
' --result-path ' + result +
' --stage ' + stage + '"',
" --container-name " + containerName +
" --result-path " + result +
" --stage " + stage + '"',
_volumes:: {
@@ -235,7 +243,7 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
commands: [
prepareTestContainer(getContainerName("smoke"), result, true),
get_build_command("run_smoke.sh") +
' --container-name ' + getContainerName("smoke"),
" --container-name " + getContainerName("smoke"),
],
},
smokelog:: {
@@ -263,14 +271,15 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
commands: [
prepareTestContainer(getContainerName("upgrade") + version, result, false),
execInnerDocker('bash -c "./upgrade_setup_' + pkg_format + '.sh '
+ version + ' '
+ result + ' '
+ arch + ' '
execInnerDocker(
'bash -c "./upgrade_setup_' + pkg_format + ".sh "
+ version + " "
+ result + " "
+ arch + " "
+ repo_pkg_url_no_res
+ ' $${UPGRADE_TOKEN}"',
getContainerName("upgrade") + version
)
),
],
},
upgradelog:: {
@@ -306,13 +315,13 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
prepareTestContainer(getContainerName("mtr"), result, true),
'MTR_SUITE_LIST=$([ "$MTR_FULL_SUITE" == true ] && echo "' + mtr_full_set + '" || echo "$MTR_SUITE_LIST")',
'apk add bash &&' +
"apk add bash &&" +
get_build_command("run_mtr.sh") +
' --container-name ' + getContainerName("mtr") +
' --distro ' + platform +
' --suite-list $${MTR_SUITE_LIST}' +
' --triggering-event ' + event
+ if std.endsWith(result, 'ASan') then ' --run-as-extern' else '',
" --container-name " + getContainerName("mtr") +
" --distro " + platform +
" --suite-list $${MTR_SUITE_LIST}" +
" --triggering-event " + event +
if std.endsWith(result, "ASan") then " --run-as-extern" else "",
],
[if (std.member(ignoreFailureStepList, "mtr")) then "failure"]: "ignore",
@@ -540,7 +549,7 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
SCCACHE_S3_KEY_PREFIX: result + branch + server + arch,
},
# errorprone if we pass --custom-cmake-flags twice, the last one will win
// errorprone if we pass --custom-cmake-flags twice, the last one will win
commands: [
"mkdir /mdb/" + builddir + "/" + result,
]
@@ -554,7 +563,7 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
"--build-path " + "/mdb/" + builddir + "/builddir " +
" " + customBootstrapParamsForExisitingPipelines(platform) +
" " + customBuildFlags(customBootstrapParamsKey) +
" | " + get_build_command("ansi2txt.sh") +
" 2>&1 | " + get_build_command("ansi2txt.sh") +
"/mdb/" + builddir + "/" + result + '/build.log "',
],
},
@@ -636,7 +645,17 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
};
local AllPipelines = [
local AllPipelines =
[
Pipeline(b, platform, triggeringEvent, a, server, flag, "")
for a in ["amd64"]
for b in std.objectFields(platforms)
for platform in ["rockylinux:8"]
for flag in ["gcc-toolset"]
for triggeringEvent in events
for server in servers[current_branch]
] +
[
Pipeline(b, p, e, a, s)
for b in std.objectFields(platforms)
for p in platforms[b]
@@ -650,6 +669,7 @@ local AllPipelines = [
for server in servers[current_branch]
for a in archs
] +
// clang
[
Pipeline(b, platform, triggeringEvent, a, server, "", buildenv)
for a in ["amd64"]
@@ -660,6 +680,15 @@ local AllPipelines = [
for server in servers[current_branch]
] +
// last argument is to ignore mtr and regression failures
[
Pipeline(b, platform, triggeringEvent, a, server, "", "", ["regression", "mtr"])
for a in ["amd64"]
for b in std.objectFields(platforms)
for platform in ["ubuntu:24.04", "rockylinux:9"]
for triggeringEvent in events
for server in extra_servers[current_branch]
] +
// // last argument is to ignore mtr and regression failures
[
Pipeline(b, platform, triggeringEvent, a, server, flag, envcommand, ["regression", "mtr"])
for a in ["amd64"]
@@ -680,15 +709,9 @@ local AllPipelines = [
for triggeringEvent in events
for server in servers[current_branch]
] +
[
Pipeline(b, platform, triggeringEvent, a, server, flag, "")
for a in ["amd64"]
for b in std.objectFields(platforms)
for platform in ["rockylinux:8"]
for flag in ["gcc-toolset"]
for triggeringEvent in events
for server in servers[current_branch]
];
[];
local FinalPipeline(branch, event) = {
kind: "pipeline",
@@ -718,7 +741,6 @@ local FinalPipeline(branch, event) = {
};
AllPipelines +
[
FinalPipeline(b, "cron")

21
build/bootstrap_mcs.sh
View File

@@ -443,7 +443,15 @@ construct_cmake_flags() {
if [[ $SCCACHE = true ]]; then
warn "Use sccache"
MDB_CMAKE_FLAGS+=(-DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache)
# Use full path to ensure sccache is found during RPM builds
MDB_CMAKE_FLAGS+=(-DCMAKE_C_COMPILER_LAUNCHER=/usr/local/bin/sccache -DCMAKE_CXX_COMPILER_LAUNCHER=/usr/local/bin/sccache)
message "Sccache binary check:"
ls -la /usr/local/bin/sccache || warn "sccache binary not found"
/usr/local/bin/sccache --version || warn "sccache version failed"
message "Starting sccache server:"
/usr/local/bin/sccache --start-server 2>&1 || warn "Failed to start sccache server"
fi
if [[ $RUN_BENCHMARKS = true ]]; then
@@ -532,7 +540,13 @@ build_package() {
cd $MDB_SOURCE_PATH
if [[ $PKG_FORMAT == "rpm" ]]; then
command="cmake ${MDB_CMAKE_FLAGS[@]} && make -j\$(nproc) package"
message "Configuring cmake for RPM package"
MDB_CMAKE_FLAGS+=(-DCPACK_PACKAGE_DIRECTORY=$MARIA_BUILD_PATH/..)
cmake "${MDB_CMAKE_FLAGS[@]}" -S"$MDB_SOURCE_PATH" -B"$MARIA_BUILD_PATH"
check_errorcode
message "Building RPM package"
command="cmake --build \"$MARIA_BUILD_PATH\" -j$(nproc) --target package"
else
export DEBIAN_FRONTEND="noninteractive"
export DEB_BUILD_OPTIONS="parallel=$(nproc)"
@@ -844,7 +858,8 @@ if [[ $BUILD_PACKAGES = true ]]; then
exit_code=$?
if [[ $SCCACHE = true ]]; then
sccache --show-adv-stats
message "Final sccache statistics:"
/usr/local/bin/sccache --show-adv-stats
fi
exit $exit_code

12
build/build_cmapi.sh
View File

@@ -41,6 +41,15 @@ on_exit() {
}
trap on_exit EXIT
get_cmapi_git_revision() {
# Prefer explicit CMAPI_GIT_REVISION; fallback to DRONE_COMMIT; otherwise read current repo revision
local rev="${CMAPI_GIT_REVISION:-${DRONE_COMMIT:-}}"
if [[ -z "$rev" ]]; then
rev="$(git -C "$COLUMNSTORE_SOURCE_PATH" rev-parse --short=12 HEAD 2>/dev/null || echo unknown)"
fi
echo "$rev"
}
install_deps() {
echo "Installing dependencies..."
@@ -90,7 +99,8 @@ install_deps() {
build_cmapi() {
cd "$COLUMNSTORE_SOURCE_PATH"/cmapi
./cleanup.sh
cmake -D"${PKG_FORMAT^^}"=1 -DSERVER_DIR="$MDB_SOURCE_PATH" . && make package
CMAPI_GIT_REVISION_ARG="$(get_cmapi_git_revision)"
cmake -D"${PKG_FORMAT^^}"=1 -DSERVER_DIR="$MDB_SOURCE_PATH" -DCMAPI_GIT_REVISION="$CMAPI_GIT_REVISION_ARG" . && make package
}
install_deps
build_cmapi

16
build/postInstall_storage_engine.sh
View File

@@ -8,3 +8,19 @@ fi
mkdir -p /var/lib/columnstore/local
columnstore-post-install --rpmmode=$rpmmode
# Attempt to load ColumnStore SELinux policy (best-effort, no hard dependency)
POLICY_PATH="/usr/share/columnstore/policy/selinux/columnstore.pp"
if command -v getenforce >/dev/null 2>&1 && command -v semodule >/dev/null 2>&1; then
MODE=$(getenforce 2>/dev/null || echo Disabled)
case "$MODE" in
Enforcing|Permissive)
if [ -r "$POLICY_PATH" ]; then
semodule -i "$POLICY_PATH" || true
fi
;;
*)
:
;;
esac
fi

7
build/preUn_storage_engine.sh
View File

@@ -10,6 +10,13 @@ fi
if [ $rpmmode = erase ]; then
columnstore-pre-uninstall
# Best-effort removal of ColumnStore SELinux policy on erase
if command -v semodule >/dev/null 2>&1; then
if semodule -l 2>/dev/null | grep -q '^columnstore\b'; then
semodule -r columnstore || true
fi
fi
fi
exit 0

22
build/prepare_test_container.sh
View File

@@ -5,10 +5,11 @@ set -eo pipefail
SCRIPT_LOCATION=$(dirname "$0")
COLUMNSTORE_SOURCE_PATH=$(realpath "$SCRIPT_LOCATION"/../)
MDB_SOURCE_PATH=$(realpath "$SCRIPT_LOCATION"/../../../..)
source "$SCRIPT_LOCATION"/utils.sh
echo "Arguments received: $@"
message "Arguments received: $@"
optparse.define short=c long=container-name desc="Name of the Docker container to run tests in" variable=CONTAINER_NAME
optparse.define short=i long=docker-image desc="Docker image name to start container from" variable=DOCKER_IMAGE
@@ -25,7 +26,7 @@ if [[ "$EUID" -ne 0 ]]; then
fi
if [[ -z "${CONTAINER_NAME:-}" || -z "${DOCKER_IMAGE:-}" || -z "${RESULT:-}" || -z "${DO_SETUP:-}" || -z "${PACKAGES_URL:-}" ]]; then
echo "Please provide --container-name, --docker-image, --result-path, --packages-url and --do-setup parameters, e.g. ./prepare_test_stage.sh --container-name smoke11212 --docker-image detravi/ubuntu:24.04 --result-path ubuntu24.04 --packages-url https://cspkg.s3.amazonaws.com/stable-23.10/pull_request/91/10.6-enterprise --do-setup true"
warn "Please provide --container-name, --docker-image, --result-path, --packages-url and --do-setup parameters, e.g. ./prepare_test_stage.sh --container-name smoke11212 --docker-image detravi/ubuntu:24.04 --result-path ubuntu24.04 --packages-url https://cspkg.s3.amazonaws.com/stable-23.10/pull_request/91/10.6-enterprise --do-setup true"
exit 1
fi
@@ -62,7 +63,7 @@ start_container() {
elif [[ "$CONTAINER_NAME" == *regression* ]]; then
docker_run_args+=(--shm-size=500m --memory 15g)
else
echo "Unknown container type: $CONTAINER_NAME"
error "Unknown container type: $CONTAINER_NAME"
exit 1
fi
@@ -126,15 +127,24 @@ prepare_container() {
execInnerDocker "$CONTAINER_NAME" 'sysctl -w kernel.core_pattern="/core/%E_${RESULT}_core_dump.%p"'
#Install columnstore in container
echo "Installing columnstore..."
message "Installing columnstore..."
SERVER_VERSION=$(grep -E 'MYSQL_VERSION_(MAJOR|MINOR)' $MDB_SOURCE_PATH/VERSION | cut -d'=' -f2 | paste -sd. -)
message "Server version of build is $SERVER_VERSION"
if [[ "$RESULT" == *rocky* ]]; then
execInnerDockerWithRetry "$CONTAINER_NAME" 'yum install -y MariaDB-columnstore-engine MariaDB-test'
else
execInnerDockerWithRetry "$CONTAINER_NAME" 'apt update -y && apt install -y mariadb-plugin-columnstore mariadb-test mariadb-test-data mariadb-plugin-columnstore-dbgsym'
execInnerDockerWithRetry "$CONTAINER_NAME" 'apt update -y && apt install -y mariadb-plugin-columnstore mariadb-test mariadb-test-data mariadb-plugin-columnstore-dbgsym mariadb-test-dbgsym'
if [[ $SERVER_VERSION == '10.6' ]]; then
execInnerDockerWithRetry "$CONTAINER_NAME" 'apt install -y mariadb-client-10.6-dbgsym mariadb-client-core-10.6-dbgsym mariadb-server-10.6-dbgsym mariadb-server-core-10.6-dbgsym'
else
execInnerDockerWithRetry "$CONTAINER_NAME" 'apt install -y mariadb-client-dbgsym mariadb-client-core-dbgsym mariadb-server-dbgsym mariadb-server-core-dbgsym'
fi
fi
sleep 5
echo "PrepareTestStage completed in $CONTAINER_NAME"
message "PrepareTestStage completed in $CONTAINER_NAME"
}
if [[ -z $(docker ps -q --filter "name=${CONTAINER_NAME}") ]]; then

28
build/selinux_policy_rpm_post.sh
View File

@@ -1,28 +0,0 @@
#!/bin/sh
# Post-install script to load ColumnStore SELinux policy if SELinux is enabled
# This script must not introduce new runtime dependencies; it only uses coreutils and typical SELinux tools if present.
set -e
POLICY_PATH="/usr/share/columnstore/policy/selinux/columnstore.pp"
# If SELinux tooling is not present, or policy file missing, silently exit
command -v getenforce >/dev/null 2>&1 || exit 0
command -v semodule >/dev/null 2>&1 || exit 0
# Only attempt to install when SELinux is enforcing or permissive
MODE=$(getenforce 2>/dev/null || echo Disabled)
case "$MODE" in
Enforcing|Permissive)
if [ -r "$POLICY_PATH" ]; then
# Install or upgrade the module; do not fail the entire package if this fails
semodule -i "$POLICY_PATH" || true
fi
;;
*)
# Disabled or unknown, do nothing
:
;;
esac
exit 0

15
build/selinux_policy_rpm_postun.sh
View File

@@ -1,15 +0,0 @@
#!/bin/sh
# Post-uninstall script to remove ColumnStore SELinux policy module if present
# No new runtime dependencies; use SELinux tools only if available.
set -e
# If SELinux tooling is not present, silently exit
command -v semodule >/dev/null 2>&1 || exit 0
# Remove the module if it is installed; do not fail package removal if this fails
if semodule -l 2>/dev/null | grep -q '^columnstore\b'; then
semodule -r columnstore || true
fi
exit 0

26
cmake/boost.cmake
View File

@@ -1,4 +1,7 @@
find_package(Boost 1.88.0 COMPONENTS chrono filesystem program_options regex system thread)
# Single source of truth for Boost components we need
set(BOOST_COMPONENTS chrono filesystem program_options regex system thread)
find_package(Boost 1.88.0 COMPONENTS ${BOOST_COMPONENTS})
if(Boost_FOUND)
add_custom_target(external_boost)
@@ -35,11 +38,20 @@ elseif(COLUMNSTORE_WITH_LIBCPP)
endif()
set(_b2args cxxflags=${_cxxargs};cflags=-fPIC;threading=multi;${_extra};toolset=${_toolset}
--without-mpi;--without-charconv;--without-python;--prefix=${INSTALL_LOCATION} linkflags=${_linkflags}
--prefix=${INSTALL_LOCATION} linkflags=${_linkflags}
)
# Derived helper strings from BOOST_COMPONENTS
set(_boost_with_libs_list ${BOOST_COMPONENTS})
string(REPLACE ";" "," _boost_with_libs_csv "${_boost_with_libs_list}")
set(_boost_b2_with_args)
foreach(_lib ${BOOST_COMPONENTS})
list(APPEND _boost_b2_with_args "--with-${_lib}")
endforeach()
string(REPLACE ";" " " _boost_b2_with_args_str "${_boost_b2_with_args}")
set(byproducts)
foreach(name chrono filesystem program_options regex system thread)
foreach(name ${BOOST_COMPONENTS})
set(lib boost_${name})
add_library(${lib} STATIC IMPORTED GLOBAL)
add_dependencies(${lib} external_boost)
@@ -50,7 +62,7 @@ endforeach()
set(LOG_BOOST_INSTEAD_OF_SCREEN "")
if(COLUMNSTORE_MAINTAINER_MODE)
set(LOG_BOOST_INSTEAD_OF_SCREEN "LOG_BUILD TRUE LOG_INSTALL TRUE")
set(LOG_BOOST_INSTEAD_OF_SCREEN "LOG_BUILD TRUE LOG_INSTALL TRUE LOG_CONFIGURE TRUE")
endif()
ExternalProject_Add(
@@ -58,13 +70,13 @@ ExternalProject_Add(
PREFIX .boost
URL https://archives.boost.io/release/1.88.0/source/boost_1_88_0.tar.gz
URL_HASH SHA256=3621533e820dcab1e8012afd583c0c73cf0f77694952b81352bf38c1488f9cb4
CONFIGURE_COMMAND ./bootstrap.sh
CONFIGURE_COMMAND ./bootstrap.sh --with-libraries=${_boost_with_libs_csv}
UPDATE_COMMAND ""
PATCH_COMMAND ${CMAKE_COMMAND} -E chdir <SOURCE_DIR> patch -p1 -i
${CMAKE_SOURCE_DIR}/storage/columnstore/columnstore/cmake/boost.1.88.named_proxy.hpp.patch
BUILD_COMMAND ./b2 -q ${_b2args}
BUILD_COMMAND ./b2 -q -d0 ${_b2args} ${_boost_b2_with_args}
BUILD_IN_SOURCE TRUE
INSTALL_COMMAND ./b2 -q install ${_b2args}
INSTALL_COMMAND ./b2 -q -d0 install ${_b2args} ${_boost_b2_with_args}
${LOG_BOOST_INSTEAD_OF_SCREEN}
EXCLUDE_FROM_ALL TRUE
${byproducts}

29
cmake/cpack_manage.cmake
View File

@@ -28,6 +28,31 @@ macro(columnstore_add_rpm_deps)
columnstore_append_for_cpack(CPACK_RPM_columnstore-engine_PACKAGE_REQUIRES ${ARGN})
endmacro()
if(RPM)
columnstore_add_rpm_deps("snappy" "jemalloc" "procps-ng" "gawk")
if(NOT COLUMNSTORE_MAINTAINER)
return()
endif()
# Columnstore-specific RPM packaging overrides 1) Use fast compression to speed up packaging
set(CPACK_RPM_COMPRESSION_TYPE
"zstd"
CACHE STRING "RPM payload compression" FORCE
)
# 2) Disable debuginfo/debugsource to avoid slow packaging and duplicate file warnings
set(CPACK_RPM_DEBUGINFO_PACKAGE
OFF
CACHE BOOL "Disable debuginfo package" FORCE
)
set(CPACK_RPM_PACKAGE_DEBUG
0
CACHE STRING "Disable RPM debug package" FORCE
)
unset(CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX CACHE)
# Ensure our overrides are applied by CPack at packaging time CPACK_PROJECT_CONFIG_FILE is included by cpack after
# CPackConfig.cmake is loaded
set(CPACK_PROJECT_CONFIG_FILE
"${CMAKE_CURRENT_LIST_DIR}/cpack_overrides.cmake"
CACHE FILEPATH "Columnstore CPack overrides" FORCE
)
columnstore_add_rpm_deps("snappy" "jemalloc" "procps-ng" "gawk")

31
cmake/cpack_overrides.cmake Normal file
View File

@@ -0,0 +1,31 @@
# Columnstore-specific CPack overrides applied at package time
# This file is referenced via CPACK_PROJECT_CONFIG_FILE and is included by CPack
# after it reads the generated CPackConfig.cmake, letting these settings win.
# Faster payload compression
set(CPACK_RPM_COMPRESSION_TYPE "zstd")
# Control debuginfo generation (symbols) without debugsource (sources)
option(CS_RPM_DEBUGINFO "Build Columnstore -debuginfo RPM (symbols only)" OFF)
if(CS_RPM_DEBUGINFO)
# Generate debuginfo RPM (symbols)
set(CPACK_RPM_DEBUGINFO_PACKAGE ON)
set(CPACK_RPM_PACKAGE_DEBUG 1)
else()
# No debuginfo RPM
set(CPACK_RPM_DEBUGINFO_PACKAGE OFF)
set(CPACK_RPM_PACKAGE_DEBUG 0)
set(CPACK_STRIP_FILES OFF)
# Prevent rpmbuild from stripping binaries and running debug post scripts.
# CPACK_STRIP_FILES only affects CPack's own stripping; rpmbuild still
# executes brp-strip and find-debuginfo by default unless we override macros.
if(DEFINED CPACK_RPM_SPEC_MORE_DEFINE)
set(CPACK_RPM_SPEC_MORE_DEFINE "${CPACK_RPM_SPEC_MORE_DEFINE}\n%define __strip /bin/true\n%define __objdump /bin/true\n%define __os_install_post %nil\n%define __debug_install_post %nil")
else()
set(CPACK_RPM_SPEC_MORE_DEFINE "%define __strip /bin/true\n%define __objdump /bin/true\n%define __os_install_post %nil\n%define __debug_install_post %nil")
endif()
endif()
# Always disable debugsource by not mapping sources
unset(CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX)

35
cmake/selinux_policy.cmake
View File

@@ -63,38 +63,3 @@ install(
COMPONENT columnstore-engine
)
# Register RPM post-install and post-uninstall scripts for the component
set(_selinux_post "${CMAKE_CURRENT_LIST_DIR}/../build/selinux_policy_rpm_post.sh")
set(_selinux_postun "${CMAKE_CURRENT_LIST_DIR}/../build/selinux_policy_rpm_postun.sh")
# POST_INSTALL: preserve existing script if set by wrapping it
if(EXISTS "${_selinux_post}")
if(DEFINED CPACK_RPM_columnstore-engine_POST_INSTALL_SCRIPT_FILE
AND CPACK_RPM_columnstore-engine_POST_INSTALL_SCRIPT_FILE
)
set(_orig_post "${CPACK_RPM_columnstore-engine_POST_INSTALL_SCRIPT_FILE}")
set(_wrap_post "${SELINUX_BUILD_DIR}/post_install_wrapper.sh")
file(WRITE "${_wrap_post}" "#!/bin/sh\n\n'${_orig_post}' \"$@\" || true\n'${_selinux_post}' \"$@\" || true\n")
execute_process(COMMAND ${CMAKE_COMMAND} -E chmod +x "${_wrap_post}")
set(CPACK_RPM_columnstore-engine_POST_INSTALL_SCRIPT_FILE "${_wrap_post}")
else()
set(CPACK_RPM_columnstore-engine_POST_INSTALL_SCRIPT_FILE "${_selinux_post}")
endif()
endif()
# POST_UNINSTALL: preserve existing script if set by wrapping it
if(EXISTS "${_selinux_postun}")
if(DEFINED CPACK_RPM_columnstore-engine_POST_UNINSTALL_SCRIPT_FILE
AND CPACK_RPM_columnstore-engine_POST_UNINSTALL_SCRIPT_FILE
)
set(_orig_postun "${CPACK_RPM_columnstore-engine_POST_UNINSTALL_SCRIPT_FILE}")
set(_wrap_postun "${SELINUX_BUILD_DIR}/post_uninstall_wrapper.sh")
file(WRITE "${_wrap_postun}"
"#!/bin/sh\n\n'${_orig_postun}' \"$@\" || true\n'${_selinux_postun}' \"$@\" || true\n"
)
execute_process(COMMAND ${CMAKE_COMMAND} -E chmod +x "${_wrap_postun}")
set(CPACK_RPM_columnstore-engine_POST_UNINSTALL_SCRIPT_FILE "${_wrap_postun}")
else()
set(CPACK_RPM_columnstore-engine_POST_UNINSTALL_SCRIPT_FILE "${_selinux_postun}")
endif()
endif()

1
cmake/thrift.cmake
View File

@@ -68,6 +68,7 @@ ExternalProject_Add(
-DCMAKE_EXE_LINKER_FLAGS=${linkflags}
-DCMAKE_SHARED_LINKER_FLAGS=${linkflags}
-DCMAKE_MODULE_LINKER_FLAGS=${linkflags}
-DCMAKE_INSTALL_MESSAGE=NEVER
BUILD_BYPRODUCTS "${THRIFT_LIBRARY_DIRS}/${CMAKE_STATIC_LIBRARY_PREFIX}thrift${CMAKE_STATIC_LIBRARY_SUFFIX}"
EXCLUDE_FROM_ALL TRUE
)

4
cmapi/.gitignore vendored
View File

@@ -87,3 +87,7 @@ result
centos8
ubuntu20.04
buildinfo.txt
# Self-signed certificates
cmapi_server/self-signed.crt
cmapi_server/self-signed.key

12
cmapi/CMakeLists.txt
View File

@@ -21,6 +21,17 @@ include(columnstore_version)
set(CMAPI_PACKAGE_VERSION "${CMAPI_VERSION_MAJOR}.${CMAPI_VERSION_MINOR}.${CMAPI_VERSION_PATCH}")
# Git revision to embed into VERSION (may be provided via -DCMAPI_GIT_REVISION or env)
if(NOT DEFINED CMAPI_GIT_REVISION OR "${CMAPI_GIT_REVISION}" STREQUAL "")
if(DEFINED ENV{CMAPI_GIT_REVISION} AND NOT "$ENV{CMAPI_GIT_REVISION}" STREQUAL "")
set(CMAPI_GIT_REVISION "$ENV{CMAPI_GIT_REVISION}")
elseif(DEFINED ENV{DRONE_COMMIT} AND NOT "$ENV{DRONE_COMMIT}" STREQUAL "")
set(CMAPI_GIT_REVISION "$ENV{DRONE_COMMIT}")
else()
set(CMAPI_GIT_REVISION "unknown")
endif()
endif()
set(CMAPI_USER "root")
set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "MariaDB ColumnStore CMAPI: cluster management API and command line tool.")
@@ -72,6 +83,7 @@ install(
cmapi_server
engine_files
mcs_cluster_tool
tracing
DESTINATION ${CMAPI_DIR}
USE_SOURCE_PERMISSIONS
PATTERN "test" EXCLUDE

1
cmapi/VERSION.template
View File

@@ -2,3 +2,4 @@ CMAPI_VERSION_MAJOR=${CMAPI_VERSION_MAJOR}
CMAPI_VERSION_MINOR=${CMAPI_VERSION_MINOR}
CMAPI_VERSION_PATCH=${CMAPI_VERSION_PATCH}
CMAPI_VERSION_RELEASE=${CMAPI_VERSION_RELEASE}
CMAPI_GIT_REVISION=${CMAPI_GIT_REVISION}

23
cmapi/cmapi_server/__main__.py
View File

@@ -16,8 +16,11 @@ from cherrypy.process import plugins
# TODO: fix dispatcher choose logic because code executing in endpoints.py
# while import process, this cause module logger misconfiguration
from cmapi_server.logging_management import config_cmapi_server_logging
from cmapi_server.sentry import maybe_init_sentry, register_sentry_cherrypy_tool
from tracing.sentry import maybe_init_sentry
from tracing.traceparent_backend import TraceparentBackend
from tracing.tracer import get_tracer
config_cmapi_server_logging()
from tracing.trace_tool import register_tracing_tools
from cmapi_server import helpers
from cmapi_server.constants import DEFAULT_MCS_CONF_PATH, CMAPI_CONF_PATH
@@ -141,10 +144,9 @@ if __name__ == '__main__':
# TODO: read cmapi config filepath as an argument
helpers.cmapi_config_check()
# Init Sentry if DSN is present
sentry_active = maybe_init_sentry()
if sentry_active:
register_sentry_cherrypy_tool()
register_tracing_tools()
get_tracer().register_backend(TraceparentBackend()) # Register default tracing backend
maybe_init_sentry() # Init Sentry if DSN is present
CertificateManager.create_self_signed_certificate_if_not_exist()
CertificateManager.renew_certificate()
@@ -153,9 +155,10 @@ if __name__ == '__main__':
root_config = {
"request.dispatch": dispatcher,
"error_page.default": jsonify_error,
# Enable tracing tools
'tools.trace.on': True,
'tools.trace_end.on': True,
}
if sentry_active:
root_config["tools.sentry.on"] = True
app.config.update({
'/': root_config,
@@ -230,10 +233,10 @@ if __name__ == '__main__':
'Something went wrong while trying to detect dbrm protocol.\n'
'Seems "controllernode" process isn\'t started.\n'
'This is just a notification, not a problem.\n'
'Next detection will started at first node\\cluster '
'Next detection will start at first node\\cluster '
'status check.\n'
f'This can cause extra {SOCK_TIMEOUT} seconds delay while\n'
'first attempt to get status.',
f'This can cause extra {SOCK_TIMEOUT} seconds delay during\n'
'this first attempt to get the status.',
exc_info=True
)
else:

9
cmapi/cmapi_server/cmapi_logger.conf
View File

@@ -7,11 +7,11 @@
},
"formatters": {
"cmapi_server": {
"format": "%(asctime)s [%(levelname)s] (%(name)s) {%(threadName)s} %(ip)s %(message)s",
"format": "%(asctime)s [%(levelname)s] (%(name)s) {%(threadName)s} %(ip)s %(message)s %(trace_params)s",
"datefmt": "%d/%b/%Y %H:%M:%S"
},
"default": {
"format": "%(asctime)s [%(levelname)s] (%(name)s) {%(threadName)s} %(message)s",
"format": "%(asctime)s [%(levelname)s] (%(name)s) {%(threadName)s} %(message)s %(trace_params)s",
"datefmt": "%d/%b/%Y %H:%M:%S"
},
"container_sh": {
@@ -75,9 +75,10 @@
"level": "DEBUG",
"propagate": false
},
"": {
"root": {
"handlers": ["console", "file"],
"level": "DEBUG"
}
}
},
"disable_existing_loggers": false
}

23
cmapi/cmapi_server/controllers/api_clients.py
View File

@@ -4,12 +4,15 @@ from typing import Any, Dict, Optional, Union
import pyotp
import requests
from cmapi_server.controllers.dispatcher import _version
from cmapi_server.constants import (
CMAPI_CONF_PATH, CURRENT_NODE_CMAPI_URL, SECRET_KEY,
CMAPI_CONF_PATH,
CURRENT_NODE_CMAPI_URL,
SECRET_KEY,
)
from cmapi_server.controllers.dispatcher import _version
from cmapi_server.exceptions import CMAPIBasicError
from cmapi_server.helpers import get_config_parser, get_current_key
from tracing.traced_session import get_traced_session
class ClusterControllerClient:
@@ -141,7 +144,7 @@ class ClusterControllerClient:
headers['Content-Type'] = 'application/json'
data = {'in_transaction': True, **(data or {})}
try:
response = requests.request(
response = get_traced_session().request(
method, url, headers=headers, json=data,
timeout=self.request_timeout, verify=False
)
@@ -151,24 +154,26 @@ class ClusterControllerClient:
except requests.HTTPError as exc:
resp = exc.response
error_msg = str(exc)
if resp.status_code == 422:
if resp is not None and resp.status_code == 422:
# in this case we think cmapi server returned some value but
# had error during running endpoint handler code
try:
resp_json = response.json()
resp_json = resp.json()
error_msg = resp_json.get('error', resp_json)
except requests.exceptions.JSONDecodeError:
error_msg = response.text
error_msg = resp.text
message = (
f'API client got an exception in request to {exc.request.url} '
f'with code {resp.status_code} and error: {error_msg}'
f'API client got an exception in request to {exc.request.url if exc.request else url} '
f'with code {resp.status_code if resp is not None else "?"} and error: {error_msg}'
)
logging.error(message)
raise CMAPIBasicError(message)
except requests.exceptions.RequestException as exc:
request_url = getattr(exc.request, 'url', url)
response_status = getattr(getattr(exc, 'response', None), 'status_code', '?')
message = (
'API client got an undefined error in request to '
f'{exc.request.url} with code {exc.response.status_code} and '
f'{request_url} with code {response_status} and '
f'error: {str(exc)}'
)
logging.error(message)

6
cmapi/cmapi_server/controllers/endpoints.py
View File

@@ -481,8 +481,8 @@ class ConfigController:
attempts = 0
# TODO: FIX IT. If got (False, False) result, for eg in case
# when there are no special CEJ user set, this check loop
# is useless and do nothing.
# when special CEJ user is not set, this check loop
# is useless and does nothing.
try:
ready, retry = system_ready(mcs_config_filename)
except CEJError as cej_error:
@@ -495,7 +495,7 @@ class ConfigController:
attempts +=1
if attempts >= 10:
module_logger.debug(
'Timed out waiting for node to be ready.'
'Timed out waiting for this node to become ready.'
)
break
time.sleep(1)

33
cmapi/cmapi_server/handlers/cluster.py
View File

@@ -4,21 +4,30 @@ from datetime import datetime
from enum import Enum
from typing import Optional
import requests
from mcs_node_control.models.misc import get_dbrm_master
from mcs_node_control.models.node_config import NodeConfig
from cmapi_server.constants import (
CMAPI_CONF_PATH, DEFAULT_MCS_CONF_PATH,
CMAPI_CONF_PATH,
DEFAULT_MCS_CONF_PATH,
)
from cmapi_server.exceptions import CMAPIBasicError
from cmapi_server.helpers import (
broadcast_new_config, get_active_nodes, get_dbroots, get_config_parser,
get_current_key, get_version, update_revision_and_manager,
broadcast_new_config,
get_active_nodes,
get_config_parser,
get_current_key,
get_dbroots,
get_version,
update_revision_and_manager,
)
from cmapi_server.node_manipulation import (
add_node, add_dbroot, remove_node, switch_node_maintenance,
add_dbroot,
add_node,
remove_node,
switch_node_maintenance,
)
from mcs_node_control.models.misc import get_dbrm_master
from mcs_node_control.models.node_config import NodeConfig
from tracing.traced_session import get_traced_session
class ClusterAction(Enum):
@@ -50,7 +59,7 @@ def toggle_cluster_state(
broadcast_new_config(config, distribute_secrets=True)
class ClusterHandler():
class ClusterHandler:
"""Class for handling MCS Cluster operations."""
@staticmethod
@@ -78,7 +87,7 @@ class ClusterHandler():
for node in active_nodes:
url = f'https://{node}:8640/cmapi/{get_version()}/node/status'
try:
r = requests.get(url, verify=False, headers=headers)
r = get_traced_session().request('GET', url, verify=False, headers=headers)
r.raise_for_status()
r_json = r.json()
if len(r_json.get('services', 0)) == 0:
@@ -277,7 +286,7 @@ class ClusterHandler():
payload['cluster_mode'] = mode
try:
r = requests.put(url, headers=headers, json=payload, verify=False)
r = get_traced_session().request('PUT', url, headers=headers, json=payload, verify=False)
r.raise_for_status()
response['cluster-mode'] = mode
except Exception as err:
@@ -330,7 +339,7 @@ class ClusterHandler():
logger.debug(f'Setting new api key to "{node}".')
url = f'https://{node}:8640/cmapi/{get_version()}/node/apikey-set'
try:
resp = requests.put(url, verify=False, json=body)
resp = get_traced_session().request('PUT', url, verify=False, json=body, headers={})
resp.raise_for_status()
r_json = resp.json()
if active_nodes_count > 0:
@@ -383,7 +392,7 @@ class ClusterHandler():
logger.debug(f'Setting new log level to "{node}".')
url = f'https://{node}:8640/cmapi/{get_version()}/node/log-level'
try:
resp = requests.put(url, verify=False, json=body)
resp = get_traced_session().request('PUT', url, verify=False, json=body, headers={})
resp.raise_for_status()
r_json = resp.json()
if active_nodes_count > 0:

30
cmapi/cmapi_server/helpers.py
View File

@@ -11,7 +11,6 @@ import os
import socket
import time
from collections import namedtuple
from functools import partial
from random import random
from shutil import copyfile
from typing import Tuple, Optional
@@ -20,6 +19,8 @@ from urllib.parse import urlencode, urlunparse
import aiohttp
import lxml.objectify
import requests
from tracing.traced_session import get_traced_session
from tracing.traced_aiohttp import create_traced_async_session
from cmapi_server.exceptions import CMAPIBasicError
# Bug in pylint https://github.com/PyCQA/pylint/issues/4584
@@ -153,9 +154,9 @@ def start_transaction(
body['timeout'] = (
final_time - datetime.datetime.now()
).seconds
r = requests.put(
url, verify=False, headers=headers, json=body,
timeout=10
r = get_traced_session().request(
'PUT', url, verify=False, headers=headers,
json=body, timeout=10
)
# a 4xx error from our endpoint;
@@ -219,8 +220,9 @@ def rollback_txn_attempt(key, version, txnid, nodes):
url = f"https://{node}:8640/cmapi/{version}/node/rollback"
for retry in range(5):
try:
r = requests.put(
url, verify=False, headers=headers, json=body, timeout=5
r = get_traced_session().request(
'PUT', url, verify=False, headers=headers,
json=body, timeout=5
)
r.raise_for_status()
except requests.Timeout:
@@ -274,7 +276,10 @@ def commit_transaction(
url = f"https://{node}:8640/cmapi/{version}/node/commit"
for retry in range(5):
try:
r = requests.put(url, verify = False, headers = headers, json = body, timeout = 5)
r = get_traced_session().request(
'PUT', url, verify=False, headers=headers,
json=body, timeout=5
)
r.raise_for_status()
except requests.Timeout as e:
logging.warning(f"commit_transaction(): timeout on node {node}")
@@ -373,7 +378,7 @@ def broadcast_new_config(
url = f'https://{node}:8640/cmapi/{version}/node/config'
resp_json: dict = dict()
async with aiohttp.ClientSession() as session:
async with create_traced_async_session() as session:
try:
async with session.put(
url, headers=headers, json=body, ssl=False, timeout=120
@@ -656,7 +661,7 @@ def get_current_config_file(
headers = {'x-api-key' : key}
url = f'https://{node}:8640/cmapi/{get_version()}/node/config'
try:
r = requests.get(url, verify=False, headers=headers, timeout=5)
r = get_traced_session().request('GET', url, verify=False, headers=headers, timeout=5)
r.raise_for_status()
config = r.json()['config']
except Exception as e:
@@ -767,14 +772,17 @@ def if_primary_restart(
success = False
while not success and datetime.datetime.now() < endtime:
try:
response = requests.put(url, verify = False, headers = headers, json = body, timeout = 60)
response = get_traced_session().request(
'PUT', url, verify=False, headers=headers,
json=body, timeout=60
)
response.raise_for_status()
success = True
except Exception as e:
logging.warning(f"if_primary_restart(): failed to start the cluster, got {str(e)}")
time.sleep(10)
if not success:
logging.error(f"if_primary_restart(): failed to start the cluster. Manual intervention is required.")
logging.error("if_primary_restart(): failed to start the cluster. Manual intervention is required.")
def get_cej_info(config_root):

29
cmapi/cmapi_server/logging_management.py
View File

@@ -7,6 +7,7 @@ import cherrypy
from cherrypy import _cperror
from cmapi_server.constants import CMAPI_LOG_CONF_PATH
from tracing.tracer import get_tracer
class AddIpFilter(logging.Filter):
@@ -16,6 +17,28 @@ class AddIpFilter(logging.Filter):
return True
def install_trace_record_factory() -> None:
"""Install a LogRecord factory that adds 'trace_params' field.
'trace_params' will be an empty string if there is no active trace/span
(like in MainThread, where there is no incoming requests).
Otherwise it will contain trace parameters.
"""
current_factory = logging.getLogRecordFactory()
def factory(*args, **kwargs): # type: ignore[no-untyped-def]
record = current_factory(*args, **kwargs)
trace_id, span_id, parent_span_id = get_tracer().current_trace_ids()
if trace_id and span_id:
record.trace_params = f'rid={trace_id} sid={span_id}'
if parent_span_id:
record.trace_params += f' psid={parent_span_id}'
else:
record.trace_params = ""
return record
logging.setLogRecordFactory(factory)
def custom_cherrypy_error(
self, msg='', context='', severity=logging.INFO, traceback=False
):
@@ -119,7 +142,10 @@ def config_cmapi_server_logging():
cherrypy._cplogging.LogManager.access_log_format = (
'{h} ACCESS "{r}" code {s}, bytes {b}, user-agent "{a}"'
)
# Ensure trace_params is available on every record
install_trace_record_factory()
dict_config(CMAPI_LOG_CONF_PATH)
disable_unwanted_loggers()
def change_loggers_level(level: str):
@@ -135,3 +161,6 @@ def change_loggers_level(level: str):
loggers.append(logging.getLogger()) # add RootLogger
for logger in loggers:
logger.setLevel(level)
def disable_unwanted_loggers():
logging.getLogger("urllib3").setLevel(logging.WARNING)

66
cmapi/cmapi_server/managers/application.py
View File

@@ -1,5 +1,5 @@
import logging
from typing import Optional
from typing import Optional, Tuple, Dict
from cmapi_server.constants import VERSION_PATH
@@ -7,23 +7,61 @@ from cmapi_server.constants import VERSION_PATH
class AppManager:
started: bool = False
version: Optional[str] = None
git_revision: Optional[str] = None
@classmethod
def get_version(cls) -> str:
"""Get CMAPI version.
:return: cmapi version
:rtype: str
"""
if cls.version:
return cls.version
with open(VERSION_PATH, encoding='utf-8') as version_file:
version = '.'.join([
i.strip().split('=')[1]
for i in version_file.read().splitlines() if i
])
if not version:
logging.error('Couldn\'t detect version from VERSION file!')
version = 'Undefined'
version, revision = cls._read_version_file()
cls.version = version
cls.git_revision = revision
return cls.version
@classmethod
def get_git_revision(cls) -> Optional[str]:
if cls.git_revision is not None:
return cls.git_revision
_, revision = cls._read_version_file()
cls.git_revision = revision
return cls.git_revision
@classmethod
def _read_version_file(cls) -> Tuple[str, Optional[str]]:
"""Read structured values from VERSION file.
Returns tuple: (semantic_version, git_revision or None)
"""
values: Dict[str, str] = {}
try:
with open(VERSION_PATH, encoding='utf-8') as version_file:
for line in version_file.read().splitlines():
if not line or '=' not in line:
continue
key, val = line.strip().split('=', 1)
values[key.strip()] = val.strip()
except Exception:
logging.exception("Failed to read VERSION file")
return 'Undefined', None
# Release (build) part is optional
release = values.get('CMAPI_VERSION_RELEASE')
revision = values.get('CMAPI_GIT_REVISION')
required_keys = (
'CMAPI_VERSION_MAJOR',
'CMAPI_VERSION_MINOR',
'CMAPI_VERSION_PATCH',
)
if not all(k in values and values[k] for k in required_keys):
logging.error("Couldn't detect version from VERSION file!")
return 'Undefined', revision
version = '.'.join([
values['CMAPI_VERSION_MAJOR'],
values['CMAPI_VERSION_MINOR'],
values['CMAPI_VERSION_PATCH'],
])
if release:
version = f"{version}.{release}"
return version, revision

13
cmapi/cmapi_server/node_manipulation.py
View File

@@ -14,15 +14,18 @@ from typing import Optional
import requests
from lxml import etree
from mcs_node_control.models.node_config import NodeConfig
from cmapi_server import helpers
from cmapi_server.constants import (
CMAPI_CONF_PATH, CMAPI_SINGLE_NODE_XML, DEFAULT_MCS_CONF_PATH, LOCALHOSTS,
CMAPI_CONF_PATH,
CMAPI_SINGLE_NODE_XML,
DEFAULT_MCS_CONF_PATH,
LOCALHOSTS,
MCS_DATA_PATH,
)
from cmapi_server.managers.network import NetworkManager
from mcs_node_control.models.node_config import NodeConfig
from tracing.traced_session import get_traced_session
PMS_NODE_PORT = '8620'
EXEMGR_NODE_PORT = '8601'
@@ -617,7 +620,9 @@ def _rebalance_dbroots(root, test_mode=False):
headers = {'x-api-key': key}
url = f"https://{node_ip}:8640/cmapi/{version}/node/new_primary"
try:
r = requests.get(url, verify = False, headers = headers, timeout = 10)
r = get_traced_session().request(
'GET', url, verify=False, headers=headers, timeout=10
)
r.raise_for_status()
r = r.json()
is_primary = r['is_primary']

197
cmapi/cmapi_server/sentry.py
View File

@@ -1,197 +0,0 @@
import logging
import socket
import cherrypy
import sentry_sdk
from sentry_sdk.integrations.aiohttp import AioHttpIntegration
from sentry_sdk.integrations.logging import LoggingIntegration
from cmapi_server import helpers
from cmapi_server.constants import CMAPI_CONF_PATH
SENTRY_ACTIVE = False
logger = logging.getLogger(__name__)
def maybe_init_sentry() -> bool:
"""Initialize Sentry from CMAPI configuration.
Reads config and initializes Sentry only if dsn parameter is present in corresponding section.
The initialization enables the following integrations:
- LoggingIntegration: capture warning-level logs as Sentry events and use
lower-level logs as breadcrumbs.
- AioHttpIntegration: propagate trace headers for outbound requests made
with `aiohttp`.
The function is a no-op if the DSN is missing.
Returns: True if Sentry is initialized, False otherwise.
"""
global SENTRY_ACTIVE
try:
cfg_parser = helpers.get_config_parser(CMAPI_CONF_PATH)
dsn = helpers.dequote(
cfg_parser.get('Sentry', 'dsn', fallback='').strip()
)
if not dsn:
return False
environment = helpers.dequote(
cfg_parser.get('Sentry', 'environment', fallback='development').strip()
)
traces_sample_rate_str = helpers.dequote(
cfg_parser.get('Sentry', 'traces_sample_rate', fallback='1.0').strip()
)
except Exception:
logger.exception('Failed to initialize Sentry.')
return False
try:
sentry_logging = LoggingIntegration(
level=logging.INFO,
event_level=logging.WARNING,
)
try:
traces_sample_rate = float(traces_sample_rate_str)
except ValueError:
logger.error('Invalid traces_sample_rate: %s', traces_sample_rate_str)
traces_sample_rate = 1.0
sentry_sdk.init(
dsn=dsn,
environment=environment,
traces_sample_rate=traces_sample_rate,
integrations=[sentry_logging, AioHttpIntegration()],
)
SENTRY_ACTIVE = True
logger.info('Sentry initialized for CMAPI via config.')
except Exception:
logger.exception('Failed to initialize Sentry.')
return False
logger.info('Sentry successfully initialized.')
return True
def _sentry_on_start_resource():
"""Start or continue a Sentry transaction for the current CherryPy request.
- Continues an incoming distributed trace using Sentry trace headers if
present; otherwise starts a new transaction with `op='http.server'`.
- Pushes the transaction into the current Sentry scope and attaches useful
request metadata as tags and context (HTTP method, path, client IP,
hostname, request ID, and a filtered subset of headers).
- Stores the transaction on the CherryPy request object for later finishing
in `_sentry_on_end_request`.
"""
if not SENTRY_ACTIVE:
return
try:
request = cherrypy.request
headers = dict(getattr(request, 'headers', {}) or {})
name = f"{request.method} {request.path_info}"
transaction = sentry_sdk.start_transaction(
op='http.server', name=name, continue_from_headers=headers
)
sentry_sdk.Hub.current.scope.set_span(transaction)
# Add request-level context/tags
scope = sentry_sdk.Hub.current.scope
scope.set_tag('http.method', request.method)
scope.set_tag('http.path', request.path_info)
scope.set_tag('client.ip', getattr(request.remote, 'ip', ''))
scope.set_tag('instance.hostname', socket.gethostname())
request_id = getattr(request, 'unique_id', None)
if request_id:
scope.set_tag('request.id', request_id)
# Optionally add headers as context without sensitive values
safe_headers = {k: v for k, v in headers.items()
if k.lower() not in {'authorization', 'x-api-key'}}
scope.set_context('headers', safe_headers)
request.sentry_transaction = transaction
except Exception:
logger.exception('Failed to start Sentry transaction.')
def _sentry_before_error_response():
"""Capture the current exception (if any) to Sentry before error response.
This hook runs when CherryPy prepares an error response. If an exception is
available in the current context, it will be sent to Sentry.
"""
if not SENTRY_ACTIVE:
return
try:
sentry_sdk.capture_exception()
except Exception:
logger.exception('Failed to capture exception to Sentry.')
def _sentry_on_end_request():
"""Finish the Sentry transaction for the current CherryPy request.
Attempts to set the HTTP status code on the active transaction and then
finishes it. If no transaction was started on this request, the function is
a no-op.
"""
if not SENTRY_ACTIVE:
return
try:
request = cherrypy.request
transaction = getattr(request, 'sentry_transaction', None)
if transaction is None:
return
status = cherrypy.response.status
try:
status_code = int(str(status).split()[0])
except Exception:
status_code = None
try:
if status_code is not None and hasattr(transaction, 'set_http_status'):
transaction.set_http_status(status_code)
except Exception:
logger.exception('Failed to set HTTP status code on Sentry transaction.')
transaction.finish()
except Exception:
logger.exception('Failed to finish Sentry transaction.')
class SentryTool(cherrypy.Tool):
"""CherryPy Tool that wires Sentry request lifecycle hooks.
The tool attaches handlers for `on_start_resource`, `before_error_response`,
and `on_end_request` in order to manage Sentry transactions and error
capture across the request lifecycle.
"""
def __init__(self):
cherrypy.Tool.__init__(self, 'on_start_resource', self._tool_callback, priority=50)
@staticmethod
def _tool_callback():
"""Attach Sentry lifecycle callbacks to the current CherryPy request."""
cherrypy.request.hooks.attach(
'on_start_resource', _sentry_on_start_resource, priority=50
)
cherrypy.request.hooks.attach(
'before_error_response', _sentry_before_error_response, priority=60
)
cherrypy.request.hooks.attach(
'on_end_request', _sentry_on_end_request, priority=70
)
def register_sentry_cherrypy_tool() -> None:
"""Register the Sentry CherryPy tool under `tools.sentry`.
This function is safe to call multiple times; failures are silently ignored
to avoid impacting the application startup.
"""
if not SENTRY_ACTIVE:
return
try:
cherrypy.tools.sentry = SentryTool()
except Exception:
logger.exception('Failed to register Sentry CherryPy tool.')

5
cmapi/mcs_cluster_tool/__main__.py
View File

@@ -56,8 +56,9 @@ app.command(
'Provides useful functions to review and troubleshoot the MCS cluster.'
)
)(tools_commands.review)
app.add_typer(
tools_commands.sentry_app, name='sentry', rich_help_panel='Tools commands', hidden=True
)
@app.command(
name='help-all', help='Show help for all commands in man page style.',
add_help_option=False

113
cmapi/mcs_cluster_tool/tools_commands.py
View File

@@ -11,10 +11,12 @@ from typing_extensions import Annotated
from cmapi_server.constants import (
MCS_DATA_PATH, MCS_SECRETS_FILENAME, REQUEST_TIMEOUT, TRANSACTION_TIMEOUT,
CMAPI_CONF_PATH,
)
from cmapi_server.controllers.api_clients import ClusterControllerClient
from cmapi_server.exceptions import CEJError
from cmapi_server.handlers.cej import CEJPasswordHandler
from cmapi_server.helpers import get_config_parser
from cmapi_server.managers.transaction import TransactionManager
from cmapi_server.process_dispatchers.base import BaseDispatcher
from mcs_cluster_tool.constants import MCS_COLUMNSTORE_REVIEW_SH
@@ -380,3 +382,114 @@ def review(
if not success:
raise typer.Exit(code=1)
raise typer.Exit(code=0)
# Sentry subcommand app
sentry_app = typer.Typer(help='Manage Sentry DSN configuration for error tracking.')
@sentry_app.command()
@handle_output
def show():
"""Show current Sentry DSN configuration."""
try:
# Read existing config
cfg_parser = get_config_parser(CMAPI_CONF_PATH)
if not cfg_parser.has_section('Sentry'):
typer.echo('Sentry is disabled (no configuration found).', color='yellow')
raise typer.Exit(code=0)
dsn = cfg_parser.get('Sentry', 'dsn', fallback='').strip().strip("'\"")
environment = cfg_parser.get('Sentry', 'environment', fallback='development').strip().strip("'\"")
if not dsn:
typer.echo('Sentry is disabled (DSN is empty).', color='yellow')
else:
typer.echo('Sentry is enabled:', color='green')
typer.echo(f' DSN: {dsn}')
typer.echo(f' Environment: {environment}')
except Exception as e:
typer.echo(f'Error reading configuration: {str(e)}', color='red')
raise typer.Exit(code=1)
raise typer.Exit(code=0)
@sentry_app.command()
@handle_output
def enable(
dsn: Annotated[
str,
typer.Argument(
help='Sentry DSN URL to enable for error tracking.',
)
],
environment: Annotated[
str,
typer.Option(
'--environment', '-e',
help='Sentry environment name (default: development).',
)
] = 'development'
):
"""Enable Sentry error tracking with the provided DSN."""
if not dsn:
typer.echo('DSN cannot be empty.', color='red')
raise typer.Exit(code=1)
try:
# Read existing config
cfg_parser = get_config_parser(CMAPI_CONF_PATH)
# Add or update Sentry section
if not cfg_parser.has_section('Sentry'):
cfg_parser.add_section('Sentry')
cfg_parser.set('Sentry', 'dsn', f"'{dsn}'")
cfg_parser.set('Sentry', 'environment', f"'{environment}'")
# Write config back to file
with open(CMAPI_CONF_PATH, 'w') as config_file:
cfg_parser.write(config_file)
typer.echo('Sentry error tracking enabled successfully.', color='green')
typer.echo(f'DSN: {dsn}', color='green')
typer.echo(f'Environment: {environment}', color='green')
typer.echo('Note: Restart cmapi service for changes to take effect.', color='yellow')
except Exception as e:
typer.echo(f'Error updating configuration: {str(e)}', color='red')
raise typer.Exit(code=1)
raise typer.Exit(code=0)
@sentry_app.command()
@handle_output
def disable():
"""Disable Sentry error tracking by removing the configuration."""
try:
# Read existing config
cfg_parser = get_config_parser(CMAPI_CONF_PATH)
if not cfg_parser.has_section('Sentry'):
typer.echo('Sentry is already disabled (no configuration found).', color='yellow')
raise typer.Exit(code=0)
# Remove the entire Sentry section
cfg_parser.remove_section('Sentry')
# Write config back to file
with open(CMAPI_CONF_PATH, 'w') as config_file:
cfg_parser.write(config_file)
typer.echo('Sentry error tracking disabled successfully.', color='green')
typer.echo('Note: Restart cmapi service for changes to take effect.', color='yellow')
except Exception as e:
typer.echo(f'Error updating configuration: {str(e)}', color='red')
raise typer.Exit(code=1)
raise typer.Exit(code=0)

13
cmapi/mcs_node_control/models/node_config.py
View File

@@ -262,15 +262,12 @@ class NodeConfig:
)
raise
def in_active_nodes(self, root):
def in_active_nodes(self, root) -> bool:
my_names = set(self.get_network_addresses_and_names())
active_nodes = [
node.text for node in root.findall("./ActiveNodes/Node")
]
for node in active_nodes:
if node in my_names:
return True
return False
active_nodes = {node.text for node in root.findall("./ActiveNodes/Node")}
am_i_active = bool(my_names.intersection(active_nodes))
module_logger.debug("Active nodes: %s, my names: %s, am i active: %s", active_nodes, my_names, am_i_active)
return am_i_active
def get_current_pm_num(self, root):
# Find this node in the Module* tags, return the module number

49
cmapi/tracing/__init__.py Normal file
View File

@@ -0,0 +1,49 @@
"""Tracing support for CMAPI
Despite having many files, the idea of this package is simple: MCS is a distributed system,
and we need to be able to trace requests across the system.
We need to understand:
* how one incoming request caused many others
* how long each request took
* which request each log line corresponds to
etc
The basic high-level mechanism is this:
1. Each incoming request is assigned a trace ID (or it may already have one, see point 2).
2. This trace ID is propagated to all other outbound requests that are caused by this request.
3. Each sub-operation is assigned a span ID. Request ID stays the same, but the span ID changes.
4. Each span can have a parent span ID, which is the span ID of the request that caused this span.
5. So basically, we have a tree of spans, and the trace ID identifies the root of the tree.
TraceID/SpanID/ParentSpanID are added to each log line, so we can identify which request each log line corresponds to.
Trace attributes are passed through the system via request headers, and here it becomes a bit more complicated.
There are two technologies that we use to pass these ids:
1. W3C TraceContext. This is a standard, it has a fixed header and its format.
The header is called `traceparent`. It encapsulates trace id and span id.
2. Sentry. For historical reasons, it has different headers. And in our setup it is optional.
But Sentry is very useful, we also use it to monitor the errors, and it has a powerful UI, so we support it too.
How is it implemented?
1. We have a global tracer object, that is used to create spans and pass them through the system.
2. It is a facade that hides two tracing backends with the same interface: TraceparentBackend and SentryBackend.
3. We have CherryPy tool that processes incoming requests, extracts traceparent header (or generates its parts),
creates a span for each request, injects traceparent header into the response.
4. For each outcoming request, we ask tracer to create a new span and to inject tracing headers into the request.
To avoid boilerplate, there is a TracedSession, an extension to requests that does all that.
For async requests, there is a TracedAsyncSession, that does the same.
5. When the request is finished, we ask tracer to finish/pop the current span.
Logging:
There is a trace record factory, that adds a new field trace_params to each log record.
trace_params is a string representation of trace id, span id and parent span id.
If in current context they are empty (like in MainThread that doesn't process requests), trace_params is an empty string.
Sentry reporting:
If Sentry is enabled, we send info about errors and exceptions into it. We also send logs that preceded the problem
as breadcrumbs to understand context of the error.
As we keep Sentry updated about the current trace and span, when an error happens, info about the trace will be sent to Sentry.
So we will know which chain of requests caused the error.
"""

37
cmapi/tracing/backend.py Normal file
View File

@@ -0,0 +1,37 @@
from abc import ABC, abstractmethod
from typing import Any, Dict, Optional
from tracing.span import TraceSpan
class TracerBackend(ABC):
@abstractmethod
def on_span_start(self, span: TraceSpan) -> None:
raise NotImplementedError
@abstractmethod
def on_span_end(self, span: TraceSpan, exc: Optional[BaseException]) -> None:
raise NotImplementedError
def on_span_event(self, span: TraceSpan, name: str, attrs: Dict[str, Any]) -> None:
return
def on_span_status(self, span: TraceSpan, code: str, description: str) -> None:
return
def on_span_exception(self, span: TraceSpan, exc: BaseException) -> None:
return
def on_span_attribute(self, span: TraceSpan, key: str, value: Any) -> None:
return
def on_inject_headers(self, headers: Dict[str, str]) -> None:
return
def on_incoming_request(self, headers: Dict[str, str], method: str, path: str) -> None:
return
def on_request_finished(self, status_code: Optional[int]) -> None:
return

76
cmapi/tracing/sentry.py Normal file
View File

@@ -0,0 +1,76 @@
import logging
import sentry_sdk
from sentry_sdk.integrations.logging import LoggingIntegration
from cmapi_server import helpers
from cmapi_server.constants import CMAPI_CONF_PATH
from tracing.sentry_backend import SentryBackend
from tracing.tracer import get_tracer
from cmapi_server.managers.application import AppManager
SENTRY_ACTIVE = False
logger = logging.getLogger(__name__)
def maybe_init_sentry() -> bool:
"""Initialize Sentry from CMAPI configuration.
Reads config and initializes Sentry only if dsn parameter is present
in corresponding section of cmapi config.
The initialization enables LoggingIntegration: it captures error-level logs as Sentry events
and uses lower-level logs as breadcrumbs.
Returns: True if Sentry is initialized, False otherwise.
"""
global SENTRY_ACTIVE
try:
cfg_parser = helpers.get_config_parser(CMAPI_CONF_PATH)
dsn = helpers.dequote(
cfg_parser.get('Sentry', 'dsn', fallback='').strip()
)
if not dsn:
return False
environment = helpers.dequote(
cfg_parser.get('Sentry', 'environment', fallback='development').strip()
)
except Exception:
logger.exception('Failed to initialize Sentry.')
return False
try:
sentry_logging = LoggingIntegration(
level=logging.INFO,
event_level=logging.ERROR,
)
# Compose release string: cmapi-version(+shortrev)
version = AppManager.get_version()
shortrev = (AppManager.get_git_revision() or '').strip()
shortrev = shortrev[:12] if shortrev else ''
release = version if not shortrev else f"{version}+{shortrev}"
sentry_sdk.init(
dsn=dsn,
environment=environment,
sample_rate=1.0,
traces_sample_rate=1.0,
integrations=[sentry_logging],
release=release,
)
# Add tags for easier filtering in Sentry
if shortrev:
sentry_sdk.set_tag("git_revision", shortrev)
sentry_sdk.set_tag("cmapi_version", version)
SENTRY_ACTIVE = True
# Register backend to mirror our internal spans into Sentry
get_tracer().register_backend(SentryBackend())
logger.info('Sentry initialized for CMAPI via config.')
except Exception:
logger.exception('Failed to initialize Sentry.')
return False
logger.info('Sentry successfully initialized.')
return True

109
cmapi/tracing/sentry_backend.py Normal file
View File

@@ -0,0 +1,109 @@
import logging
import contextvars
from typing import Any, Dict, Optional
import sentry_sdk
from sentry_sdk.tracing import Transaction
from tracing.tracer import TraceSpan, TracerBackend
from tracing.utils import swallow_exceptions
logger = logging.getLogger(__name__)
class SentryBackend(TracerBackend):
"""Mirror spans and events from our Tracer into Sentry SDK."""
def __init__(self) -> None:
self._active_spans: Dict[str, Any] = {}
self._current_transaction = contextvars.ContextVar[Optional[Transaction]]("sentry_transaction", default=None)
@swallow_exceptions
def on_span_start(self, span: TraceSpan) -> None:
kind_to_op = {
'SERVER': 'http.server',
'CLIENT': 'http.client',
'INTERNAL': 'internal',
}
op = kind_to_op.get(span.kind.upper(), 'internal')
sdk_span = sentry_sdk.start_span(op=op, description=span.name)
sdk_span.set_tag('w3c.trace_id', span.trace_id)
sdk_span.set_tag('w3c.span_id', span.span_id)
if span.parent_span_id:
sdk_span.set_tag('w3c.parent_span_id', span.parent_span_id)
if span.attributes:
sdk_span.set_data('cmapi.span_attributes', dict(span.attributes))
sdk_span.__enter__()
self._active_spans[span.span_id] = sdk_span
@swallow_exceptions
def on_span_end(self, span: TraceSpan, exc: Optional[BaseException]) -> None:
sdk_span = self._active_spans.pop(span.span_id, None)
if sdk_span is None:
return
if exc is not None:
sdk_span.set_status('internal_error')
sdk_span.__exit__(
type(exc) if exc else None,
exc,
exc.__traceback__ if exc else None
)
@swallow_exceptions
def on_span_event(self, span: TraceSpan, name: str, attrs: Dict[str, Any]) -> None:
sentry_sdk.add_breadcrumb(category='event', message=name, data=dict(attrs))
@swallow_exceptions
def on_span_status(self, span: TraceSpan, code: str, description: str) -> None:
sdk_span = self._active_spans.get(span.span_id)
if sdk_span is not None:
sdk_span.set_status(code)
@swallow_exceptions
def on_span_exception(self, span: TraceSpan, exc: BaseException) -> None:
sentry_sdk.capture_exception(exc)
@swallow_exceptions
def on_span_attribute(self, span: TraceSpan, key: str, value: Any) -> None:
sdk_span = self._active_spans.get(span.span_id)
if sdk_span is not None:
sdk_span.set_data(f'attr.{key}', value)
@swallow_exceptions
def on_inject_headers(self, headers: Dict[str, str]) -> None:
traceparent = sentry_sdk.get_traceparent()
baggage = sentry_sdk.get_baggage()
if traceparent:
headers['sentry-trace'] = traceparent
if baggage:
headers['baggage'] = baggage
@swallow_exceptions
def on_incoming_request(self, headers: Dict[str, str], method: str, path: str) -> None:
name = f"{method} {path}" if method or path else "http.server"
# Continue from incoming headers, then START the transaction per SDK v2
continued = sentry_sdk.continue_trace(headers or {}, op='http.server', name=name)
transaction = sentry_sdk.start_transaction(transaction=continued)
# Store started transaction in context var and make current (enter)
self._current_transaction.set(transaction)
transaction.__enter__()
scope = sentry_sdk.Hub.current.scope
if method:
scope.set_tag('http.method', method)
if path:
scope.set_tag('http.path', path)
@swallow_exceptions
def on_request_finished(self, status_code: Optional[int]) -> None:
transaction = self._current_transaction.get()
if transaction is None:
return
if status_code is not None:
transaction.set_http_status(status_code)
# Exit to restore parent and finish the transaction
transaction.__exit__(None, None, None)
# Clear transaction in this context
self._current_transaction.set(None)

46
cmapi/tracing/span.py Normal file
View File

@@ -0,0 +1,46 @@
from typing import TYPE_CHECKING
from dataclasses import dataclass
from typing import Any, Dict
from tracing.utils import swallow_exceptions
if TYPE_CHECKING:
from tracing.tracer import Tracer
@dataclass
class TraceSpan:
"""Span handle bound to a tracer.
Provides helpers to add attributes/events/status/exception that
will never propagate exceptions.
"""
name: str
kind: str # "SERVER" | "CLIENT" | "INTERNAL"
start_ns: int
trace_id: str
span_id: str
parent_span_id: str
attributes: Dict[str, Any]
tracer: "Tracer"
@swallow_exceptions
def set_attribute(self, key: str, value: Any) -> None:
self.attributes[key] = value
self.tracer._notify_attribute(self, key, value)
@swallow_exceptions
def add_event(self, name: str, **attrs: Any) -> None:
self.tracer._notify_event(self, name, attrs)
@swallow_exceptions
def set_status(self, code: str, description: str = "") -> None:
self.attributes["status.code"] = code
if description:
self.attributes["status.description"] = description
self.tracer._notify_status(self, code, description)
@swallow_exceptions
def record_exception(self, exc: BaseException) -> None:
self.tracer._notify_exception(self, exc)

67
cmapi/tracing/trace_tool.py Normal file
View File

@@ -0,0 +1,67 @@
"""
CherryPy tool that uses the tracer to start a span for each request.
"""
import socket
from typing import Dict
import cherrypy
from tracing.tracer import get_tracer
def _on_request_start() -> None:
req = cherrypy.request
tracer = get_tracer()
headers: Dict[str, str] = dict(req.headers or {})
tracer.notify_incoming_request(
headers=headers,
method=getattr(req, 'method', ''),
path=getattr(req, 'path_info', '')
)
trace_id, parent_span_id = tracer.extract_traceparent(headers)
tracer.set_incoming_context(trace_id, parent_span_id)
span_name = f"{getattr(req, 'method', 'HTTP')} {getattr(req, 'path_info', '/')}"
ctx = tracer.start_as_current_span(span_name, kind="SERVER")
span = ctx.__enter__()
span.set_attribute('http.method', getattr(req, 'method', ''))
span.set_attribute('http.path', getattr(req, 'path_info', ''))
span.set_attribute('client.ip', getattr(getattr(req, 'remote', None), 'ip', ''))
span.set_attribute('instance.hostname', socket.gethostname())
safe_headers = {k: v for k, v in headers.items() if k.lower() not in {'authorization', 'x-api-key'}}
span.set_attribute('sentry.incoming_headers', safe_headers)
req._trace_span_ctx = ctx
req._trace_span = span
tracer.inject_traceparent(cherrypy.response.headers) # type: ignore[arg-type]
def _on_request_end() -> None:
req = cherrypy.request
try:
status_str = str(cherrypy.response.status)
status_code = int(status_str.split()[0])
except Exception:
status_code = None
tracer = get_tracer()
tracer.notify_request_finished(status_code)
span = getattr(req, "_trace_span", None)
if span is not None and status_code is not None:
span.set_attribute('http.status_code', status_code)
ctx = getattr(req, "_trace_span_ctx", None)
if ctx is not None:
try:
ctx.__exit__(None, None, None)
finally:
req._trace_span_ctx = None
req._trace_span = None
def register_tracing_tools() -> None:
cherrypy.tools.trace = cherrypy.Tool("on_start_resource", _on_request_start, priority=10)
cherrypy.tools.trace_end = cherrypy.Tool("on_end_resource", _on_request_end, priority=80)

40
cmapi/tracing/traced_aiohttp.py Normal file
View File

@@ -0,0 +1,40 @@
"""Async sibling of TracedSession."""
from typing import Any
import aiohttp
from tracing.tracer import get_tracer
class TracedAsyncSession(aiohttp.ClientSession):
async def _request(
self, method: str, str_or_url: Any, *args: Any, **kwargs: Any
) -> aiohttp.ClientResponse:
tracer = get_tracer()
headers = kwargs.get("headers") or {}
if headers is None:
headers = {}
kwargs["headers"] = headers
url_text = str(str_or_url)
span_name = f"HTTP {method} {url_text}"
with tracer.start_as_current_span(span_name, kind="CLIENT") as span:
span.set_attribute("http.method", method)
span.set_attribute("http.url", url_text)
tracer.inject_outbound_headers(headers)
try:
response = await super()._request(method, str_or_url, *args, **kwargs)
except Exception as exc:
span.set_status("ERROR", str(exc))
raise
else:
span.set_attribute("http.status_code", response.status)
return response
def create_traced_async_session(**kwargs: Any) -> TracedAsyncSession:
return TracedAsyncSession(**kwargs)

44
cmapi/tracing/traced_session.py Normal file
View File

@@ -0,0 +1,44 @@
"""Customized requests.Session that automatically traces outbound HTTP calls."""
from typing import Any, Optional
import requests
from tracing.tracer import get_tracer
class TracedSession(requests.Session):
def request(self, method: str, url: str, *args: Any, **kwargs: Any) -> requests.Response:
tracer = get_tracer()
headers = kwargs.get("headers") or {}
if headers is None:
headers = {}
kwargs["headers"] = headers
span_name = f"HTTP {method} {url}"
with tracer.start_as_current_span(span_name, kind="CLIENT") as span:
span.set_attribute("http.method", method)
span.set_attribute("http.url", url)
tracer.inject_outbound_headers(headers)
try:
response = super().request(method, url, *args, **kwargs)
except Exception as exc:
span.set_status("ERROR", str(exc))
raise
else:
span.set_attribute("http.status_code", response.status_code)
return response
_default_session: Optional[TracedSession] = None
def get_traced_session() -> TracedSession:
global _default_session
if _default_session is None:
_default_session = TracedSession()
return _default_session

38
cmapi/tracing/traceparent_backend.py Normal file
View File

@@ -0,0 +1,38 @@
import logging
import time
from typing import Any, Dict, Optional
from tracing.tracer import TracerBackend, TraceSpan
from tracing.utils import swallow_exceptions
logger = logging.getLogger("tracing")
class TraceparentBackend(TracerBackend):
"""Default backend that logs span lifecycle and mirrors events/status."""
@swallow_exceptions
def on_span_start(self, span: TraceSpan) -> None:
logger.info(
"span_begin name=%s kind=%s trace_id=%s span_id=%s parent=%s attrs=%s",
span.name, span.kind, span.trace_id, span.span_id,
span.parent_span_id, span.attributes,
)
@swallow_exceptions
def on_span_end(self, span: TraceSpan, exc: Optional[BaseException]) -> None:
duration_ms = (time.time_ns() - span.start_ns) / 1_000_000
logger.info(
"span_end name=%s kind=%s trace_id=%s span_id=%s parent=%s duration_ms=%.3f attrs=%s",
span.name, span.kind, span.trace_id, span.span_id,
span.parent_span_id, duration_ms, span.attributes,
)
@swallow_exceptions
def on_span_event(self, span: TraceSpan, name: str, attrs: Dict[str, Any]) -> None:
logger.info(
"span_event name=%s trace_id=%s span_id=%s attrs=%s",
name, span.trace_id, span.span_id, attrs,
)

155
cmapi/tracing/tracer.py Normal file
View File

@@ -0,0 +1,155 @@
"""This module implements a tracer facade that creates spans, injects/extracts traceparent headers,
and delegates span lifecycle and enrichment to pluggable backends (e.g., Traceparent and Sentry).
It uses contextvars to store the trace/span/parent_span ids and start time for each context.
"""
import contextvars
import logging
import time
from collections.abc import Iterator
from contextlib import contextmanager
from typing import Any, Dict, List, Optional, Tuple
from tracing.backend import TracerBackend
from tracing.span import TraceSpan
from tracing.utils import (
rand_16_hex,
rand_8_hex,
format_traceparent,
parse_traceparent,
)
logger = logging.getLogger(__name__)
# Context vars are something like thread-local storage, they are context-local variables
_current_trace_id = contextvars.ContextVar[str]("trace_id", default="")
_current_span_id = contextvars.ContextVar[str]("span_id", default="")
_current_parent_span_id = contextvars.ContextVar[str]("parent_span_id", default="")
_current_span_start_ns = contextvars.ContextVar[int]("span_start_ns", default=0)
class Tracer:
def __init__(self) -> None:
self._backends: List[TracerBackend] = []
def register_backend(self, backend: TracerBackend) -> None:
try:
self._backends.append(backend)
logger.info(
"Tracing backend registered: %s", backend.__class__.__name__
)
except Exception:
logger.exception("Failed to register tracing backend")
def clear_backends(self) -> None:
self._backends.clear()
@contextmanager
def start_as_current_span(self, name: str, kind: str = "INTERNAL") -> Iterator[TraceSpan]:
trace_id = _current_trace_id.get() or rand_16_hex()
parent_span = _current_span_id.get()
new_span_id = rand_8_hex()
tok_tid = _current_trace_id.set(trace_id)
tok_sid = _current_span_id.set(new_span_id)
tok_pid = _current_parent_span_id.set(parent_span)
tok_start = _current_span_start_ns.set(time.time_ns())
span = TraceSpan(
name=name,
kind=kind,
start_ns=_current_span_start_ns.get(),
trace_id=trace_id,
span_id=new_span_id,
parent_span_id=parent_span,
attributes={"span.kind": kind, "span.name": name},
tracer=self,
)
caught_exc: Optional[BaseException] = None
try:
for backend in list(self._backends):
backend.on_span_start(span)
yield span
except BaseException as exc:
span.record_exception(exc)
span.set_status("ERROR", str(exc))
caught_exc = exc
raise
finally:
for backend in list(self._backends):
backend.on_span_end(span, caught_exc)
_current_span_start_ns.reset(tok_start)
_current_parent_span_id.reset(tok_pid)
_current_span_id.reset(tok_sid)
_current_trace_id.reset(tok_tid)
def set_incoming_context(
self,
trace_id: Optional[str] = None,
parent_span_id: Optional[str] = None,
) -> None:
if trace_id:
_current_trace_id.set(trace_id)
if parent_span_id:
_current_parent_span_id.set(parent_span_id)
def current_trace_ids(self) -> Tuple[str, str, str]:
return _current_trace_id.get(), _current_span_id.get(), _current_parent_span_id.get()
def inject_traceparent(self, headers: Dict[str, str]) -> None:
trace_id, span_id, _ = self.current_trace_ids()
if not trace_id or not span_id:
trace_id = trace_id or rand_16_hex()
span_id = span_id or rand_8_hex()
headers["traceparent"] = format_traceparent(trace_id, span_id)
def inject_outbound_headers(self, headers: Dict[str, str]) -> None:
self.inject_traceparent(headers)
for backend in list(self._backends):
backend.on_inject_headers(headers)
def notify_incoming_request(self, headers: Dict[str, str], method: str, path: str) -> None:
for backend in list(self._backends):
backend.on_incoming_request(headers, method, path)
def notify_request_finished(self, status_code: Optional[int]) -> None:
for backend in list(self._backends):
backend.on_request_finished(status_code)
def extract_traceparent(self, headers: Dict[str, str]) -> Tuple[str, str]:
raw_traceparent = (headers.get("traceparent")
or headers.get("Traceparent")
or headers.get("TRACEPARENT"))
if not raw_traceparent:
return "", ""
parsed = parse_traceparent(raw_traceparent)
if not parsed:
return "", ""
return parsed[0], parsed[1]
def _notify_event(self, span: TraceSpan, name: str, attrs: Dict[str, Any]) -> None:
for backend in list(self._backends):
backend.on_span_event(span, name, attrs)
def _notify_status(self, span: TraceSpan, code: str, description: str) -> None:
for backend in list(self._backends):
backend.on_span_status(span, code, description)
def _notify_exception(self, span: TraceSpan, exc: BaseException) -> None:
for backend in list(self._backends):
backend.on_span_exception(span, exc)
def _notify_attribute(self, span: TraceSpan, key: str, value: Any) -> None:
for backend in list(self._backends):
backend.on_span_attribute(span, key, value)
_tracer = Tracer()
def get_tracer() -> Tracer:
return _tracer

54
cmapi/tracing/utils.py Normal file
View File

@@ -0,0 +1,54 @@
import logging
import os
from functools import wraps
from typing import Optional, Tuple
logger = logging.getLogger("tracing")
def swallow_exceptions(method):
"""Decorator that logs exceptions and prevents them from propagating up."""
@wraps(method)
def _wrapper(*args, **kwargs):
try:
return method(*args, **kwargs)
except Exception:
logger.exception("%s failed", getattr(method, "__qualname__", repr(method)))
return None
return _wrapper
def rand_16_hex() -> str:
"""Return 16 random bytes as a 32-char hex string (trace_id size)."""
return os.urandom(16).hex()
def rand_8_hex() -> str:
"""Return 8 random bytes as a 16-char hex string (span_id size)."""
return os.urandom(8).hex()
def format_traceparent(trace_id: str, span_id: str, flags: str = "01") -> str:
"""Build a W3C traceparent header (version 00)."""
return f"00-{trace_id}-{span_id}-{flags}"
def parse_traceparent(header: str) -> Optional[Tuple[str, str, str]]:
"""Parse W3C traceparent and return (trace_id, span_id, flags) or None."""
try:
parts = header.strip().split("-")
if len(parts) != 4 or parts[0] != "00":
logger.error("Invalid traceparent: %s", header)
return None
trace_id, span_id, flags = parts[1], parts[2], parts[3]
if len(trace_id) != 32 or len(span_id) != 16 or len(flags) != 2:
return None
# W3C: all zero trace_id/span_id are invalid
if set(trace_id) == {"0"} or set(span_id) == {"0"}:
return None
return trace_id, span_id, flags
except Exception:
logger.exception("Failed to parse traceparent: %s", header)
return None

21
dbcon/mysql/is_columnstore_columns.cpp
View File

@@ -62,12 +62,25 @@ static int is_columnstore_columns_fill(THD* thd, TABLE_LIST* tables, COND* cond)
InformationSchemaCond isCond;
execplan::CalpontSystemCatalog csc;
const std::vector<
std::pair<execplan::CalpontSystemCatalog::OID, execplan::CalpontSystemCatalog::TableName> >
catalog_tables = csc.getTables();
// Use FE path for syscat queries issued from mysqld
csc.identity(execplan::CalpontSystemCatalog::FE);
std::vector<
std::pair<execplan::CalpontSystemCatalog::OID, execplan::CalpontSystemCatalog::TableName> >
catalog_tables;
try
{
catalog_tables = csc.getTables("", lower_case_table_names);
}
catch (IDBExcept&)
{
return 1;
}
catch (std::exception&)
{
return 1;
}
if (cond)
{
isCond.getCondItems(cond);

1
mysql-test/columnstore/autopilot/t/mcs4010_autopilot_cross_engine_join.test
View File

@@ -4,6 +4,7 @@
# Author: Daniel Lee, daniel.lee@mariadb.com
# -------------------------------------------------------------- #
#
--source ../include/disable_for_11.4_and_later.inc
--source ../include/have_columnstore.inc
--source ../include/detect_maxscale.inc
#

50
mysql-test/columnstore/basic/r/mcol_2000.result
View File

@@ -20,17 +20,17 @@ l date,
m datetime,
o time,
s char(17) character set utf8,
t varchar(17) character set utf8mb4,
t varchar(17) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin,
w blob(10),
x tinyblob,
y blob,
z mediumblob,
aa longblob,
bb text(17) character set utf8,
cc tinytext character set utf8mb4,
dd text character set utf8mb4,
ee mediumtext character set utf8mb4,
ff longtext character set utf8mb4
cc tinytext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin,
dd text CHARACTER SET utf8mb4 COLLATE utf8mb4_bin,
ee mediumtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin,
ff longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin
) default charset=koi8r ENGINE=InnoDB;
create table copy1 like orig;
alter table copy1 engine=columnstore;
@@ -53,17 +53,17 @@ orig CREATE TABLE `orig` (
`m` datetime DEFAULT NULL,
`o` time DEFAULT NULL,
`s` char(17) CHARACTER SET utf8mb3 DEFAULT NULL,
`t` varchar(17) CHARACTER SET utf8mb4 DEFAULT NULL,
`t` varchar(17) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL,
`w` tinyblob DEFAULT NULL,
`x` tinyblob DEFAULT NULL,
`y` blob DEFAULT NULL,
`z` mediumblob DEFAULT NULL,
`aa` longblob DEFAULT NULL,
`bb` tinytext CHARACTER SET utf8mb3 DEFAULT NULL,
`cc` tinytext CHARACTER SET utf8mb4 DEFAULT NULL,
`dd` text CHARACTER SET utf8mb4 DEFAULT NULL,
`ee` mediumtext CHARACTER SET utf8mb4 DEFAULT NULL,
`ff` longtext CHARACTER SET utf8mb4 DEFAULT NULL
`cc` tinytext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL,
`dd` text CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL,
`ee` mediumtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL,
`ff` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL
) ENGINE=InnoDB DEFAULT CHARSET=koi8r
show create table copy1;
Table Create Table
@@ -82,17 +82,17 @@ copy1 CREATE TABLE `copy1` (
`m` datetime DEFAULT NULL,
`o` time DEFAULT NULL,
`s` char(17) CHARACTER SET utf8mb3 DEFAULT NULL,
`t` varchar(17) CHARACTER SET utf8mb4 DEFAULT NULL,
`t` varchar(17) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL,
`w` tinyblob DEFAULT NULL,
`x` tinyblob DEFAULT NULL,
`y` blob DEFAULT NULL,
`z` mediumblob DEFAULT NULL,
`aa` longblob DEFAULT NULL,
`bb` tinytext CHARACTER SET utf8mb3 DEFAULT NULL,
`cc` tinytext CHARACTER SET utf8mb4 DEFAULT NULL,
`dd` text CHARACTER SET utf8mb4 DEFAULT NULL,
`ee` mediumtext CHARACTER SET utf8mb4 DEFAULT NULL,
`ff` longtext CHARACTER SET utf8mb4 DEFAULT NULL
`cc` tinytext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL,
`dd` text CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL,
`ee` mediumtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL,
`ff` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL
) ENGINE=Columnstore DEFAULT CHARSET=koi8r
show create table copy2;
Table Create Table
@@ -111,17 +111,17 @@ copy2 CREATE TABLE `copy2` (
`m` datetime DEFAULT NULL,
`o` time DEFAULT NULL,
`s` char(17) CHARACTER SET utf8mb3 DEFAULT NULL,
`t` varchar(17) CHARACTER SET utf8mb4 DEFAULT NULL,
`t` varchar(17) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL,
`w` tinyblob DEFAULT NULL,
`x` tinyblob DEFAULT NULL,
`y` blob DEFAULT NULL,
`z` mediumblob DEFAULT NULL,
`aa` longblob DEFAULT NULL,
`bb` tinytext CHARACTER SET utf8mb3 DEFAULT NULL,
`cc` tinytext CHARACTER SET utf8mb4 DEFAULT NULL,
`dd` text CHARACTER SET utf8mb4 DEFAULT NULL,
`ee` mediumtext CHARACTER SET utf8mb4 DEFAULT NULL,
`ff` longtext CHARACTER SET utf8mb4 DEFAULT NULL
`cc` tinytext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL,
`dd` text CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL,
`ee` mediumtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL,
`ff` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL
) ENGINE=Columnstore DEFAULT CHARSET=koi8r
show create table copy3;
Table Create Table
@@ -140,17 +140,17 @@ copy3 CREATE TABLE `copy3` (
`m` datetime DEFAULT NULL,
`o` time DEFAULT NULL,
`s` char(17) DEFAULT NULL,
`t` varchar(17) CHARACTER SET utf8mb4 DEFAULT NULL,
`t` varchar(17) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL,
`w` tinyblob DEFAULT NULL,
`x` tinyblob DEFAULT NULL,
`y` blob DEFAULT NULL,
`z` mediumblob DEFAULT NULL,
`aa` longblob DEFAULT NULL,
`bb` tinytext DEFAULT NULL,
`cc` tinytext CHARACTER SET utf8mb4 DEFAULT NULL,
`dd` text CHARACTER SET utf8mb4 DEFAULT NULL,
`ee` mediumtext CHARACTER SET utf8mb4 DEFAULT NULL,
`ff` longtext CHARACTER SET utf8mb4 DEFAULT NULL
`cc` tinytext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL,
`dd` text CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL,
`ee` mediumtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL,
`ff` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL
) ENGINE=Columnstore DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_general_ci
drop table orig;
drop table copy1;

10
mysql-test/columnstore/basic/t/mcol_2000.test
View File

@@ -46,17 +46,17 @@ create table orig (a integer not null,
m datetime,
o time,
s char(17) character set utf8,
t varchar(17) character set utf8mb4,
t varchar(17) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin,
w blob(10),
x tinyblob,
y blob,
z mediumblob,
aa longblob,
bb text(17) character set utf8,
cc tinytext character set utf8mb4,
dd text character set utf8mb4,
ee mediumtext character set utf8mb4,
ff longtext character set utf8mb4
cc tinytext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin,
dd text CHARACTER SET utf8mb4 COLLATE utf8mb4_bin,
ee mediumtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin,
ff longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin
) default charset=koi8r ENGINE=InnoDB;
create table copy1 like orig;

1
mysql-test/columnstore/devregression/t/mcs7134_regression_bug3997.test
View File

@@ -4,6 +4,7 @@
# Author: Daniel Lee, daniel.lee@mariadb.com
# -------------------------------------------------------------- #
#
--source ../include/disable_for_11.4_and_later.inc
--source ../include/have_columnstore.inc
#
USE tpch1;

1
mysql-test/columnstore/devregression/t/mcs7135_regression_bug3998.test
View File

@@ -4,6 +4,7 @@
# Author: Daniel Lee, daniel.lee@mariadb.com
# -------------------------------------------------------------- #
#
--source ../include/disable_for_11.4_and_later.inc
--source ../include/have_columnstore.inc
#
USE tpch1;

1
mysql-test/columnstore/devregression/t/mcs7136_regression_bug4027.test
View File

@@ -4,6 +4,7 @@
# Author: Daniel Lee, daniel.lee@mariadb.com
# -------------------------------------------------------------- #
#
--source ../include/disable_for_11.4_and_later.inc
--source ../include/have_columnstore.inc
#
USE tpch1;

4
mysql-test/columnstore/include/disable_for_11.4_and_later.inc Normal file
View File

@@ -0,0 +1,4 @@
if (`SELECT (sys.version_major(), sys.version_minor(), sys.version_patch()) >= (11, 4, 0)`)
{
skip Known multiupdate bug;
}

1
tests/CMakeLists.txt
View File

@@ -22,6 +22,7 @@ if(WITH_UNITTESTS)
GIT_REPOSITORY https://github.com/google/googletest
GIT_TAG release-1.12.0
CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${EXTERNAL_INSTALL_LOCATION} -DBUILD_SHARED_LIBS=ON
-DCMAKE_INSTALL_MESSAGE=NEVER
-DCMAKE_CXX_FLAGS:STRING=${cxxflags} -DCMAKE_EXE_LINKER_FLAGS=${linkflags}
-DCMAKE_SHARED_LINKER_FLAGS=${linkflags} -DCMAKE_MODULE_LINKER_FLAGS=${linkflags}
)