From 4fbf5ff78ddb0de4f6f909a88a6edeb2cc9a871c Mon Sep 17 00:00:00 2001 From: Allen Herrera <82840027+mariadb-AllenHerrera@users.noreply.github.com> Date: Thu, 21 Dec 2023 19:42:22 -0500 Subject: [PATCH] Update cs package manager dev ubuntu (#3060) * cs_package_manager.sh - fix ubuntu dev installs * cs_package_manager.sh - remove some useless comments * debian/ubuntu dev install to use drone repo + remove wget dependancy * + exits if repo files missing * auto install aws cli for dev builds * removed commented section from cs_package_manager --- extra/cs_package_manager.sh | 184 ++++++++++++++++++++++++++++++------ 1 file changed, 153 insertions(+), 31 deletions(-) diff --git a/extra/cs_package_manager.sh b/extra/cs_package_manager.sh index 3554daac4..0409b40a5 100644 --- a/extra/cs_package_manager.sh +++ b/extra/cs_package_manager.sh @@ -192,7 +192,13 @@ do_apt_remove() { DEBIAN_FRONTEND=noninteractive apt remove --purge -y mariadb-* print_and_delete "/var/lib/mysql/*" else - apt remove mariadb-* -y + if ! apt remove mariadb-columnstore-cmapi -y; then + printf "[!!] Failed to remove columnstore \n" + fi + + if ! apt remove mariadb-* -y; then + printf "[!!] Failed to remove the rest of mariadb \n\n" + fi fi fi @@ -250,7 +256,8 @@ check_package_managers() { check_operating_system() { distro_info=$(awk -F= '/^ID=/{gsub(/"/, "", $2); print $2}' /etc/os-release) - version_id=$(grep 'VERSION_ID=' /etc/os-release | awk -F= '{gsub(/"/, "", $2); print $2}' | awk -F. '{print $1}') + version_id_exact=$( grep 'VERSION_ID=' /etc/os-release | awk -F= '{gsub(/"/, "", $2); print $2}') + version_id=$( echo "$version_id_exact" | awk -F. '{print $1}') echo "Distro: $distro_info" echo "Version: $version_id" @@ -261,13 +268,13 @@ check_operating_system() { distro="${distro_info}${version_id}" ;; debian ) - distro="${distro_info}${version_id}" + distro="${distro_info}${version_id_exact}" ;; rocky ) distro="rockylinux${version_id}" ;; ubuntu ) - distro="${distro_info}${version_id}" + distro="${distro_info}${version_id_exact}" ;; *) # unknown option printf "\ncheck_operating_system: unknown os & version: $distro_info\n" @@ -328,13 +335,48 @@ check_aws_cli_installed() { if ! command -v aws &> /dev/null ; then echo "[!] aws cli - binary could not be found" echo "[+] Installing aws cli ..." - rm -rf aws awscliv2.zip - yum install unzip -y; - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"; - unzip awscliv2.zip; - sudo ./aws/install; - mv /usr/local/bin/aws /usr/bin/aws; - aws configure set default.s3.max_concurrent_requests 700 + + cli_url="https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" + case $architecture in + x86_64 ) + cli_url="https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" + ;; + aarch64 ) + cli_url="https://awscli.amazonaws.com/awscli-exe-linux-aarch64.zip" + ;; + *) # unknown option + echo "Error: Unsupported architecture ($architecture)" + esac + + case $distro_info in + centos | rocky ) + rm -rf aws awscliv2.zip + yum install unzip -y; + curl "$cli_url" -o "awscliv2.zip"; + unzip -q awscliv2.zip; + sudo ./aws/install; + mv /usr/local/bin/aws /usr/bin/aws; + aws configure set default.s3.max_concurrent_requests 70 + ;; + ubuntu | debian ) + rm -rf aws awscliv2.zip + if ! sudo apt install unzip -y; then + echo "[!!] Installing Unzip Failed: Trying update" + sudo apt update -y; + sudo apt install unzip -y; + fi + curl "$cli_url" -o "awscliv2.zip"; + unzip -q awscliv2.zip; + sudo ./aws/install; + mv /usr/local/bin/aws /usr/bin/aws; + aws configure set default.s3.max_concurrent_requests 70 + ;; + *) # unknown option + printf "\nos & version not implemented: $distro_info\n" + exit 2; + esac + + fi } @@ -438,7 +480,8 @@ enterprise_install() { # Download Repo setup script rm -rf mariadb_es_repo_setup - wget $url ;chmod +x mariadb_es_repo_setup; + curl -LO "$url" -o mariadb_es_repo_setup; + chmod +x mariadb_es_repo_setup; if ! bash mariadb_es_repo_setup --token="$enterprise_token" --apply --mariadb-server-version="$version"; then printf "\n[!] Failed to apply mariadb_es_repo_setup...\n\n" exit 2; @@ -447,6 +490,8 @@ enterprise_install() { case $distro_info in centos | rocky ) + if [ ! -f "/etc/yum.repos.d/mariadb.repo" ]; then printf "\n[!] Expected to find mariadb.repo in /etc/yum.repos.d \n\n"; exit 1; fi; + if $enterprise_staging; then sed -i 's/mariadb-es-main/mariadb-es-staging/g' /etc/yum.repos.d/mariadb.repo sed -i 's/mariadb-enterprise-server/mariadb-enterprise-staging/g' /etc/yum.repos.d/mariadb.repo @@ -457,6 +502,8 @@ enterprise_install() { ;; ubuntu | debian ) + if [ ! -f "/etc/apt/sources.list.d/mariadb.list" ]; then printf "\n[!] Expected to find mariadb.list in /etc/apt/sources.list.d \n\n"; exit 1; fi; + if $enterprise_staging; then sed -i 's/mariadb-enterprise-server/mariadb-enterprise-staging/g' /etc/apt/sources.list.d/mariadb.list apt update @@ -721,22 +768,42 @@ dev_install() { branch="$3" build="$4" product="10.6-enterprise" + if [ -z "$branch" ]; then printf "Missing branch: $branch\n"; exit 2; fi; + if [ -z "$build" ]; then printf "Missing build: $branch\n"; exit 2; fi; # Construct URLs - s3_path="$dronePath/$branch/$build/$product/$arch/$distro" - replace="https://$dev_drone_key.s3.amazonaws.com/" - # Use sed to replace the s3 path to create https path - yum_http=$(echo "$s3_path" | sed "s|s3://$dev_drone_key/|$replace|") + s3_path="$dronePath/$branch/$build/$product/$arch" + drone_http=$(echo "$s3_path" | sed "s|s3://$dev_drone_key/|https://${dev_drone_key}.s3.amazonaws.com/|") echo "Locations:" - echo "RPM: $s3_path" - echo "Yum: $yum_http" + echo "Bucket: $s3_path" + echo "Drone: $drone_http" echo "###################################" check_dev_build_exists + + case $distro_info in + centos | rocky ) + s3_path="${s3_path}/$distro" + drone_http="${drone_http}/$distro" + do_dev_yum_install "$@" + ;; + ubuntu | debian ) + do_dev_apt_install "$@" + ;; + *) # unknown option + printf "\nos & version not implemented: $distro_info\n" + exit 2; + esac + + add_primary_node_cmapi +} + +do_dev_yum_install() { + echo "[drone] name=Drone Repository -baseurl="$yum_http" +baseurl="$drone_http" gpgcheck=0 enabled=1 " > /etc/yum.repos.d/drone.repo @@ -753,13 +820,73 @@ enabled=1 exit 1 fi - yum install MariaDB-server-*.rpm -y - yum install MariaDB-columnstore-engine MariaDB-columnstore-cmapi jq -y - systemctl start mariadb - systemctl start mariadb-columnstore-cmapi - mariadb -e "show status like '%Columnstore%';" + # Install MariaDB Server + if ! yum install MariaDB-server-*.rpm -y; then + printf "\n[!] Failed to install MariaDB-server \n\n" + exit 1; + fi + + # Install Columnstore + if ! yum install MariaDB-columnstore-engine -y; then + printf "\n[!] Failed to install columnstore \n\n" + exit 1; + fi + + # Install Cmapi + if ! yum install MariaDB-columnstore-cmapi jq -y; then + printf "\n[!] Failed to install cmapi\n\n" + exit 1; + else + systemctl start mariadb + systemctl enable mariadb-columnstore-cmapi + systemctl start mariadb-columnstore-cmapi + mariadb -e "show status like '%Columnstore%';" + sleep 2 + + add_primary_node_cmapi + fi +} + +do_dev_apt_install() { + + echo "deb [trusted=yes] ${drone_http} ${distro}/" > /etc/apt/sources.list.d/repo.list + cat << EOF > /etc/apt/preferences +Package: * +Pin: origin cspkg.s3.amazonaws.com +Pin-Priority: 1700 +EOF + + # Install MariaDB + apt-get clean + apt-get update + if ! apt install mariadb-server -y --quiet; then + printf "\n[!] Failed to install mariadb-server \n\n" + exit 1; + fi + sleep 2 + systemctl daemon-reload + systemctl enable mariadb + systemctl start mariadb + + # Install Columnstore + if ! apt install mariadb-plugin-columnstore -y --quiet; then + printf "\n[!] Failed to install columnstore \n\n" + exit 1; + fi; + + if ! apt install mariadb-columnstore-cmapi jq -y --quiet ; then + printf "\n[!] Failed to install cmapi \n\n" + mariadb -e "show status like '%Columnstore%';" + else + systemctl daemon-reload + systemctl enable mariadb-columnstore-cmapi + systemctl start mariadb-columnstore-cmapi + mariadb -e "show status like '%Columnstore%';" + sleep 2 + + add_primary_node_cmapi + fi - add_primary_node_cmapi } do_install() { @@ -980,12 +1107,7 @@ global_dependencies() { if ! command -v curl &> /dev/null; then printf "\n[!] curl not found. Please install curl\n\n" exit 1; - fi - - if ! command -v wget &> /dev/null; then - printf "\n[!] wget not found. Please install wget\n\n" - exit 1; - fi + fi } check_set_es_token() {