diff --git a/.drone.jsonnet b/.drone.jsonnet index 2c4541c62..3858b7cd3 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -396,23 +396,13 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", name: "dockerfile", depends_on: ["publish pkg", "publish cmapi build"], image: "alpine/git:2.49.0", + volumes: [pipeline._volumes.mdb], environment: { - DOCKER_BRANCH_REF: "${DRONE_SOURCE_BRANCH}", - DOCKER_REF_AUX: branch_ref, + DRONE_SOURCE_BRANCH: "${DRONE_SOURCE_BRANCH}", }, commands: [ - // compute branch. - 'echo "$$DOCKER_REF"', - 'echo "$$DOCKER_BRANCH_REF"', - // if DOCKER_REF is empty, try to see whether docker repository has a branch named as one we PR. - 'export DOCKER_REF=$${DOCKER_REF:-$$(git ls-remote https://github.com/mariadb-corporation/mariadb-columnstore-docker --h --sort origin "refs/heads/$$DOCKER_BRANCH_REF" | grep -E -o "[^/]+$$")}', - 'echo "$$DOCKER_REF"', - // DOCKER_REF can be empty if there is no appropriate branch in docker repository. - // assign what is appropriate by default. - "export DOCKER_REF=$${DOCKER_REF:-$$DOCKER_REF_AUX}", - 'echo "$$DOCKER_REF"', - "git clone --branch $$DOCKER_REF --depth 1 https://github.com/mariadb-corporation/mariadb-columnstore-docker docker", - "touch docker/.secrets", + "apk add bash && " + + get_build_command("clone_docker_repo.sh"), ], }, dockerhub:: { @@ -485,7 +475,6 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", MCS_IMAGE_NAME: "mariadb/enterprise-columnstore-dev:" + container_tags[0], }, commands: [ - "echo $$DOCKER_PASSWORD | docker login --username $$DOCKER_LOGIN --password-stdin", "apk add bash && " + get_build_command("run_multi_node_mtr.sh") + " --columnstore-image-name $${MCS_IMAGE_NAME} " + diff --git a/build/clone_docker_repo.sh b/build/clone_docker_repo.sh new file mode 100755 index 000000000..4daeae3cc --- /dev/null +++ b/build/clone_docker_repo.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +set -eo pipefail + +SCRIPT_LOCATION=$(dirname "$0") +source "$SCRIPT_LOCATION"/utils.sh + + +# Determine which branch to use +DOCKER_REF="${DOCKER_REF:-}" +DOCKER_BRANCH_REF="${DRONE_SOURCE_BRANCH:-stable-23.10}" +DOCKER_REF_AUX="stable-23.10" + +message "Checking for docker repository branch: $DOCKER_BRANCH_REF" + +# Try to find matching branch in docker repository +if [[ -z "$DOCKER_REF" ]]; then + DOCKER_REF=$(git ls-remote https://github.com/mariadb-corporation/mariadb-columnstore-docker --heads --sort origin "refs/heads/$DOCKER_BRANCH_REF" 2>/dev/null | grep -E -o "[^/]+$" || true) +fi + +# Fall back to default branch if no match found +if [[ -z "$DOCKER_REF" ]]; then + DOCKER_REF="$DOCKER_REF_AUX" +fi + +message "Cloning docker repository from branch: $DOCKER_REF" + +# Remove existing docker folder if it exists to avoid git clone conflicts +if [[ -d "docker" ]]; then + message "Removing existing docker folder..." + rm -rf docker +fi + +# Clone the docker repository +git clone --branch "$DOCKER_REF" --depth 1 https://github.com/mariadb-corporation/mariadb-columnstore-docker docker + +# Create empty .secrets file +touch docker/.secrets + +message "Docker repository cloned successfully" diff --git a/build/run_multi_node_mtr.sh b/build/run_multi_node_mtr.sh index 19829bd92..0739bd065 100755 --- a/build/run_multi_node_mtr.sh +++ b/build/run_multi_node_mtr.sh @@ -5,6 +5,7 @@ set -eo pipefail SCRIPT_LOCATION=$(dirname "$0") source "$SCRIPT_LOCATION"/utils.sh +#usage example: sudo ./run_multi_node_mtr.sh --columnstore-image-name mariadb/enterprise-columnstore-dev:stable-23.10-pull-request2240 --distro rockylinux:8 optparse.define short=i long=columnstore-image-name desc="Name of columnstore docker image" variable=MCS_IMAGE_NAME optparse.define short=d long=distro desc="Linux distro for which multinode mtr is executed" variable=DISTRO source $(optparse.build) @@ -35,12 +36,76 @@ fi message "Running multinode mtr tests..." +# Detect docker compose command (new syntax vs old) +if command -v docker &> /dev/null && docker compose version &> /dev/null; then + DOCKER_COMPOSE="docker compose" +elif command -v docker-compose &> /dev/null; then + DOCKER_COMPOSE="docker-compose" +else + error "Docker compose is not installed!" + exit 1 +fi + +# Handle Docker Hub authentication +message "Checking Docker Hub authentication..." +if [[ -n "${DOCKER_LOGIN:-}" ]] && [[ -n "${DOCKER_PASSWORD:-}" ]]; then + # CI environment - use credentials from secrets + message "Using Docker credentials from environment variables" + echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_LOGIN" --password-stdin + if [[ $? -ne 0 ]]; then + error "Docker login failed with provided credentials" + exit 1 + fi +else + message "No Docker credentials in env variables found, checking if already logged in..." + + # Try to verify docker login by checking auth for docker.io + if docker system info 2>&1 | grep -q "Username:"; then + message "Already logged in to Docker Hub" + else + message "Not logged in to Docker Hub" + message "You need to login to Docker Hub to pull enterprise-columnstore-dev images" + message "" + + # Prompt for username + read -p "Docker Hub Username: " docker_username + + # Run docker login with username (will prompt for password) + docker login -u "$docker_username" + + if [[ $? -ne 0 ]]; then + error "Docker login failed or was cancelled" + exit 1 + fi + + message "Docker login successful" + fi +fi + +# Check if docker folder exists and has required files +if [[ ! -d "docker" ]] || [[ ! -f "docker/Dockerfile" ]]; then + message "Docker folder not found or incomplete, cloning mariadb-columnstore-docker repository..." + "$SCRIPT_LOCATION"/clone_docker_repo.sh +fi + cd docker + +# Clean up any existing containers from previous runs +if docker ps -a --format '{{.Names}}' | grep -q -E '^(mcs1|mcs2|mcs3)$'; then + message "Found existing containers from previous run, cleaning up..." + $DOCKER_COMPOSE down -v 2>/dev/null || true + docker rm -f mcs1 mcs2 mcs3 2>/dev/null || true + message "Cleanup completed" +fi + cp .env_example .env sed -i "/^MCS_IMAGE_NAME=/s|=.*|=${MCS_IMAGE_NAME}|" .env sed -i "/^MAXSCALE=/s|=.*|=false|" .env -docker-compose up -d +message "Starting containers..." +$DOCKER_COMPOSE up -d + +message "Provisioning cluster..." docker exec mcs1 provision mcs1 mcs2 mcs3 docker cp ../mysql-test/columnstore mcs1:"${MTR_PATH}/suite/" docker exec -t mcs1 chown -R mysql:mysql "${MTR_PATH}"