diff --git a/build/bootstrap_mcs.sh b/build/bootstrap_mcs.sh new file mode 100755 index 000000000..f17741653 --- /dev/null +++ b/build/bootstrap_mcs.sh @@ -0,0 +1,305 @@ +#!/bin/bash + +# This script compiles/installs MCS from scratch and it makes some assumptions: +# - the server's source code is two directories above the MCS engine source. +# - the script is to be run under root. + + + +SCRIPT_LOCATION=$(dirname "$0") +MDB_SOURCE_PATH=$(realpath $SCRIPT_LOCATION/../../../..) + +source $SCRIPT_LOCATION/utils.sh + + +if [ "$EUID" -ne 0 ] + then error "Please run script as root to install MariaDb to system paths" + exit 1 +fi + +message "Building Mariadb Server from $color_yellow$MDB_SOURCE_PATH$color_normal" + +BUILD_TYPE_OPTIONS=("Debug" "RelWithDebInfo") +DISTRO_OPTIONS=("Ubuntu" "CentOS" "Debian" "openSUSE" "Rocky") +BRANCHES=($(git branch --list --no-color| grep "[^* ]+" -Eo)) + +optparse.define short=t long=build-type desc="Build Type: ${BUILD_TYPE_OPTIONS[*]}" variable=MCS_BUILD_TYPE +optparse.define short=d long=distro desc="Choouse your OS: ${DISTRO_OPTIONS[*]}" variable=OS +optparse.define short=s long=skip-deps desc="Skip install dependences" variable=SKIP_DEPS default=false value=true +optparse.define short=C long=force-cmake-reconfig desc="Force cmake reconfigure" variable=FORCE_CMAKE_CONFIG default=false value=true +optparse.define short=S long=skip-columnstore-submodules desc="Skip columnstore submodules initialization" variable=SKIP_SUBMODULES default=false value=true +optparse.define short=b long=branch desc="Choouse git branch ('none' for menu)" variable=BRANCH + +source $( optparse.build ) + +if [[ ! " ${BUILD_TYPE_OPTIONS[*]} " =~ " ${MCS_BUILD_TYPE} " ]]; then + getChoice -q "Select your Build Type" -o BUILD_TYPE_OPTIONS + MCS_BUILD_TYPE=$selectedChoice +fi + +if [[ ! " ${DISTRO_OPTIONS[*]} " =~ " ${OS} " || $OS = "CentOS" ]]; then + detect_distro +fi + +INSTALL_PREFIX="/usr/" +DATA_DIR="/var/lib/mysql/data" +CMAKE_BIN_NAME=cmake + +select_branch() +{ + if [[ ! " ${BRANCHES[*]} " =~ " ${BRANCH} " ]]; then + if [[ $BRANCH = 'none' ]]; then + getChoice -q "Select your branch" -o BRANCHES + BRANCH=$selectedChoice + fi + cd $SCRIPT_LOCATION + message "Selecting $BRANCH branch for Columnstore" + git checkout $BRANCH + cd - + + message "Turning off Columnstore submodule auto update via gitconfig" + cd $MDB_SOURCE_PATH + git config submodule.storage/columnstore/columnstore.update none + cd - + fi + + cd $SCRIPT_LOCATION + CURRENT_BRANCH=$(git branch --show-current) + cd - + message "Columnstore will be built from $color_yellow$CURRENT_BRANCH$color_normal branch" +} + +install_deps() +{ + message "Installing deps" + if [[ $OS = 'Ubuntu' || $OS = 'Debian' ]]; then + sudo apt-get -y update + apt-get -y install build-essential automake libboost-all-dev bison cmake \ + libncurses5-dev libaio-dev libsystemd-dev libpcre2-dev \ + libperl-dev libssl-dev libxml2-dev libkrb5-dev flex libpam-dev git \ + libsnappy-dev libcurl4-openssl-dev libgtest-dev libcppunit-dev googletest libsnappy-dev libjemalloc-dev + elif [[ $OS = 'CentOS' || $OS = 'Rocky' ]]; then + yum -y install epel-release \ + && yum -y groupinstall "Development Tools" \ + && yum config-manager --set-enabled powertools \ + && yum -y install bison ncurses-devel readline-devel perl-devel openssl-devel libxml2-devel gperf libaio-devel libevent-devel tree wget pam-devel snappy-devel libicu \ + && yum -y install vim wget strace ltrace gdb rsyslog net-tools openssh-server expect boost perl-DBI libicu boost-devel initscripts jemalloc-devel libcurl-devel gtest-devel cppunit-devel systemd-devel + if [[ "$OS_VERSION" == "7" ]]; then + yum -y install cmake3 + CMAKE_BIN_NAME=cmake3 + else + yum -y install cmake + fi + if [ $OS = 'Rocky' ]; then + yum install -y checkpolicy + fi + elif [ $OS = 'openSUSE' ]; then + zypper install -y bison ncurses-devel readline-devel libopenssl-devel cmake libxml2-devel gperf libaio-devel libevent-devel python-devel ruby-devel tree wget pam-devel snappy-devel libicu-devel \ + && zypper install -y libboost_system-devel libboost_filesystem-devel libboost_thread-devel libboost_regex-devel libboost_date_time-devel libboost_chrono-devel libboost_atomic-devel \ + && zypper install -y vim wget strace ltrace gdb rsyslog net-tools expect perl-DBI libicu boost-devel jemalloc-devel libcurl-devel liblz4-devel lz4 \ + && zypper install -y gcc gcc-c++ git automake libtool gtest cppunit-devel pcre2-devel systemd-devel libaio-devel snappy-devel cracklib-devel policycoreutils-devel ncurses-devel \ + && zypper install -y make flex libcurl-devel automake libtool rpm-build lsof iproute pam-devel perl-DBI expect createrepo + fi +} + +stop_service() +{ + message "Stopping MariaDB services" + systemctl stop mariadb + systemctl stop mariadb-columnstore +} + +check_service() +{ + if systemctl is-active --quiet $1; then + message "$1 service started$color_green OK $color_normal" + else + error "$1 service failed" + service $1 status + fi +} + +start_service() +{ + message "Starting MariaDB services" + systemctl start mariadb-columnstore + systemctl start mariadb + + check_service mariadb-columnstore + check_service mariadb +} + +clean_old_installation() +{ + message "Cleaning old installation" + rm -rf /var/lib/columnstore/data1/* + rm -rf /var/lib/columnstore/data/ + rm -rf /var/lib/columnstore/local/ + rm -f /var/lib/columnstore/storagemanager/storagemanager-lock + rm -f /var/lib/columnstore/storagemanager/cs-initialized + rm -rf /tmp/* + rm -rf /var/lib/mysql + rm -rf /var/run/mysqld + rm -rf $DATA_DIR + rm -rf /etc/mysql + rm -r /etc/my.cnf.d/provider_* +} + +build() +{ + message "Building sources in $color_yellow$MCS_BUILD_TYPE$color_normal mode" + + local MDB_CMAKE_FLAGS="-DWITH_SYSTEMD=yes + -DPLUGIN_COLUMNSTORE=YES + -DPLUGIN_MROONGA=NO + -DPLUGIN_ROCKSDB=NO + -DPLUGIN_TOKUDB=NO + -DPLUGIN_CONNECT=NO + -DPLUGIN_SPIDER=NO + -DPLUGIN_OQGRAPH=NO + -DPLUGIN_SPHINX=NO + -DWITH_EMBEDDED_SERVER=OFF + -DBUILD_CONFIG=mysql_release + -DWITH_WSREP=OFF + -DWITH_SSL=system + -DWITH_UNITTESTS=YES + -DWITH_BRM_UT=YES + -DCMAKE_INSTALL_PREFIX:PATH=$INSTALL_PREFIX" + + cd $MDB_SOURCE_PATH + + if [[ $SKIP_SUBMODULES = true ]] ; then + warn "Skipping initialization of columnstore submodules" + else + message "Initialization of columnstore submodules" + cd storage/columnstore/columnstore + git submodule update --init + cd - + fi + + if [[ $FORCE_CMAKE_CONFIG = true ]] ; then + warn "Erasing cmake cache" + rm -f "$MDB_SOURCE_PATH/CMakeCache.txt" + rm -rf "$MDB_SOURCE_PATH/CMakeFiles" + fi + + if [[ "$OS" = 'Ubuntu' || "$OS" = 'Debian' ]]; then + MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DDEB=bionic" + elif [ $OS = 'CentOS' ]; then + MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DRPM=CentOS7" + elif [ $OS = 'Rocky' ]; then + MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DRPM=CentOS7" + elif [ $OS = 'openSUSE' ]; then + MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DRPM=sles15" + fi + + message "building with flags $MDB_CMAKE_FLAGS" + + local CPUS=$(getconf _NPROCESSORS_ONLN) + ${CMAKE_BIN_NAME} . -DCMAKE_BUILD_TYPE=$MCS_BUILD_TYPE $MDB_CMAKE_FLAGS && \ + make -j $CPUS install + + if [ $? -ne 0 ]; then + error "!!!! BUILD FAILED" + exit 1 + fi + cd - +} + + +check_user_and_group() +{ + if [ -z "$(grep mysql /etc/passwd)" ]; then + message "Adding user mysql into /etc/passwd" + useradd -r -U mysql -d /var/lib/mysql + fi + + if [ -z "$(grep mysql /etc/group)" ]; then + GroupID = `awk -F: '{uid[$3]=1}END{for(x=100; x<=999; x++) {if(uid[x] != ""){}else{print x; exit;}}}' /etc/group` + message "Adding group mysql with id $GroupID" + groupadd -g GroupID mysql + fi +} + +install() +{ + message "Installing MariaDB" + + check_user_and_group + + mkdir -p /etc/my.cnf.d + + bash -c 'echo "[client-server] +socket=/run/mysqld/mysqld.sock" > /etc/my.cnf.d/socket.cnf' + + mv $INSTALL_PREFIX/lib/mysql/plugin/ha_columnstore.so /tmp/ha_columnstore_1.so || mv $INSTALL_PREFIX/lib64/mysql/plugin/ha_columnstore.so /tmp/ha_columnstore_2.so + message "Running mysql_install_db" + mysql_install_db --rpm --user=mysql + mv /tmp/ha_columnstore_1.so $INSTALL_PREFIX/lib/mysql/plugin/ha_columnstore.so || mv /tmp/ha_columnstore_2.so $INSTALL_PREFIX/lib64/mysql/plugin/ha_columnstore.so + + mkdir -p /etc/columnstore + + cp $MDB_SOURCE_PATH/storage/columnstore/columnstore/oam/etc/Columnstore.xml /etc/columnstore/Columnstore.xml + cp $MDB_SOURCE_PATH/storage/columnstore/columnstore/storage-manager/storagemanager.cnf /etc/columnstore/storagemanager.cnf + + cp $MDB_SOURCE_PATH/support-files/*.service /lib/systemd/system/ + cp $MDB_SOURCE_PATH/storage/columnstore/columnstore/oam/install_scripts/*.service /lib/systemd/system/ + + if [[ "$OS" = 'Ubuntu' || "$OS" = 'Debian' ]]; then + mkdir -p /usr/share/mysql + mkdir -p /etc/mysql/ + cp $MDB_SOURCE_PATH/debian/additions/debian-start.inc.sh /usr/share/mysql/debian-start.inc.sh + cp $MDB_SOURCE_PATH/debian/additions/debian-start /etc/mysql/debian-start + > /etc/mysql/debian.cnf + fi + + systemctl daemon-reload + + if [ -d "/etc/mysql/mariadb.conf.d/" ]; then + message "Copying configs from /etc/mysql/mariadb.conf.d/ to /etc/my.cnf.d" + cp -rp /etc/mysql/mariadb.conf.d/* /etc/my.cnf.d + fi + + if [ -d "/etc/mysql/conf.d/" ]; then + message "Copying configs from /etc/mysql/conf.d/ to /etc/my.cnf.d" + cp -rp /etc/mysql/conf.d/* /etc/my.cnf.d + fi + + mkdir -p /var/lib/columnstore/data1 + mkdir -p /var/lib/columnstore/data1/systemFiles + mkdir -p /var/lib/columnstore/data1/systemFiles/dbrm + mkdir -p /run/mysqld/ + + mkdir -p $DATA_DIR + chown -R mysql:mysql $DATA_DIR + chown -R mysql:mysql /var/lib/columnstore/ + chown -R mysql:mysql /run/mysqld/ + + chmod +x $INSTALL_PREFIX/bin/mariadb* + + ldconfig + + message "Running columnstore-post-install" + mkdir -p /var/lib/columnstore/local + columnstore-post-install --rpmmode=install + message "Running install_mcs_mysql" + install_mcs_mysql.sh + + chown -R syslog:syslog /var/log/mariadb/ + chmod 777 /var/log/mariadb/ + chmod 777 /var/log/mariadb/columnstore +} + + +select_branch + +if [[ $SKIP_DEPS = false ]] ; then + install_deps +fi + +stop_service +clean_old_installation +build +install +start_service +message "$color_green FINISHED $color_normal" diff --git a/build/utils.sh b/build/utils.sh new file mode 100644 index 000000000..133bfb036 --- /dev/null +++ b/build/utils.sh @@ -0,0 +1,377 @@ +color_normal=$(tput sgr0) +color_bold=$(tput bold) +color_red="$color_bold$(tput setaf 1)" +color_green=$(tput setaf 2) +color_fawn=$(tput setaf 3); color_beige="$color_fawn" +color_yellow="$color_bold$color_fawn" +color_darkblue=$(tput setaf 4) +color_blue="$color_bold$color_darkblue" +color_purple=$(tput setaf 5); color_magenta="$color_purple" +color_pink="$color_bold$color_purple" +color_darkcyan=$(tput setaf 6) +color_cyan="$color_bold$color_darkcyan" +color_gray=$(tput setaf 7) +color_darkgray="$color_bold"$(tput setaf 0) +color_white="$color_bold$color_gray" + +message() +{ + echo $color_cyan -- $@$color_normal +} + +warn() +{ + echo $color_yellow -- $@$color_normal +} + +error() +{ + echo $color_red -- $@$color_normal +} + +detect_distro() +{ + if [ -f /etc/os-release ]; then + . /etc/os-release + export OS=$NAME + export OS_VERSION=$VERSION_ID + elif type lsb_release >/dev/null 2>&1; then + # linuxbase.org + export OS=$(lsb_release -si) + export OS_VERSION=$(lsb_release -sr) + elif [ -f /etc/lsb-release ]; then + # For some versions of Debian/Ubuntu without lsb_release command + . /etc/lsb-release + export OS=$DISTRIB_ID + OS_VERSION=$DISTRIB_RELEASE + elif [ -f /etc/debian_version ]; then + # Older Debian/Ubuntu/etc. + OS=Debian + OS_VERSION=$(cat /etc/debian_version) + else + # Fall back to uname, e.g. "Linux ", also works for BSD, etc. + OS=$(uname -s) + OS_VERSION=$(uname -r) + fi + OS=$(echo $OS | cut -f 1 -d " ") + message "Detected $color_yellow$OS $OS_VERSION$color_normal" +} + +menuStr="" + +function hideCursor(){ + printf "\033[?25l" + + # capture CTRL+C so cursor can be reset + trap "showCursor && exit 0" 2 +} + +function showCursor(){ + printf "\033[?25h" +} + +function clearLastMenu(){ + local msgLineCount=$(printf "$menuStr" | wc -l) + # moves the curser up N lines so the output overwrites it + echo -en "\033[${msgLineCount}A" + + # clear to end of screen to ensure there's no text left behind from previous input + [ $1 ] && tput ed +} + +function renderMenu(){ + local start=0 + local selector="" + local instruction="$1" + local selectedIndex=$2 + local listLength=$itemsLength + local longest=0 + local spaces="" + menuStr="\n $instruction\n" + + # Get the longest item from the list so that we know how many spaces to add + # to ensure there's no overlap from longer items when a list is scrolling up or down. + for (( i=0; i<$itemsLength; i++ )); do + if (( ${#menuItems[i]} > longest )); then + longest=${#menuItems[i]} + fi + done + spaces=$(printf ' %.0s' $(eval "echo {1.."$(($longest))"}")) + + if [ $3 -ne 0 ]; then + listLength=$3 + + if [ $selectedIndex -ge $listLength ]; then + start=$(($selectedIndex+1-$listLength)) + listLength=$(($selectedIndex+1)) + fi + fi + + for (( i=$start; i<$listLength; i++ )); do + local currItem="${menuItems[i]}" + currItemLength=${#currItem} + + if [[ $i = $selectedIndex ]]; then + selectedChoice="${currItem}" + selector="${color_green}ᐅ${color_normal}" + currItem="${color_green}${currItem}${color_normal}" + else + selector=" " + fi + + currItem="${spaces:0:0}${currItem}${spaces:currItemLength}" + + menuStr="${menuStr}\n ${selector} ${currItem}" + done + + menuStr="${menuStr}\n" + + # whether or not to overwrite the previous menu output + [ $4 ] && clearLastMenu + + printf "${menuStr}" +} + +function getChoice(){ + local KEY__ARROW_UP=$(echo -e "\033[A") + local KEY__ARROW_DOWN=$(echo -e "\033[B") + local KEY__ENTER=$(echo -e "\n") + local captureInput=true + local displayHelp=false + local maxViewable=0 + local instruction="Select an item from the list:" + local selectedIndex=0 + + remainingArgs=() + while [[ $# -gt 0 ]]; do + key="$1" + + case $key in + -h|--help) + displayHelp=true + shift + ;; + -i|--index) + selectedIndex=$2 + shift 2 + ;; + -m|--max) + maxViewable=$2 + shift 2 + ;; + -o|--options) + menuItems=$2[@] + menuItems=("${!menuItems}") + shift 2 + ;; + -q|--query) + instruction="$2" + shift 2 + ;; + *) + remainingArgs+=("$1") + shift + ;; + esac + done + + # just display help + if $displayHelp; then + echo; + echo "Usage: getChoice [OPTION]..." + echo "Renders a keyboard navigable menu with a visual indicator of what's selected." + echo; + echo " -h, --help Displays this message" + echo " -i, --index The initially selected index for the options" + echo " -m, --max Limit how many options are displayed" + echo " -o, --options An Array of options for a User to choose from" + echo " -q, --query Question or statement presented to the User" + echo; + echo "Example:" + echo " foodOptions=(\"pizza\" \"burgers\" \"chinese\" \"sushi\" \"thai\" \"italian\" \"shit\")" + echo; + echo " getChoice -q \"What do you feel like eating?\" -o foodOptions -i \$((\${#foodOptions[@]}-1)) -m 4" + echo " printf \"\\n First choice is '\${selectedChoice}'\\n\"" + echo; + echo " getChoice -q \"Select another option in case the first isn't available\" -o foodOptions" + echo " printf \"\\n Second choice is '\${selectedChoice}'\\n\"" + echo; + + return 0 + fi + + set -- "${remainingArgs[@]}" + local itemsLength=${#menuItems[@]} + + # no menu items, at least 1 required + if [[ $itemsLength -lt 1 ]]; then + printf "\n [ERROR] No menu items provided\n" + exit 1 + fi + + renderMenu "$instruction" $selectedIndex $maxViewable + hideCursor + + while $captureInput; do + read -rsn3 key # `3` captures the escape (\033'), bracket ([), & type (A) characters. + + case "$key" in + "$KEY__ARROW_UP") + selectedIndex=$((selectedIndex-1)) + (( $selectedIndex < 0 )) && selectedIndex=$((itemsLength-1)) + + renderMenu "$instruction" $selectedIndex $maxViewable true + ;; + + "$KEY__ARROW_DOWN") + selectedIndex=$((selectedIndex+1)) + (( $selectedIndex == $itemsLength )) && selectedIndex=0 + + renderMenu "$instruction" $selectedIndex $maxViewable true + ;; + + "$KEY__ENTER") + clearLastMenu true + showCursor + captureInput=false + ;; + esac + done +} + + +function optparse.throw_error(){ + local message="$1" + error "OPTPARSE: ERROR: $message" + exit 1 +} + +# ----------------------------------------------------------------------------------------------------------------------------- +function optparse.define(){ + if [ $# -lt 3 ]; then + optparse.throw_error "optparse.define [] [] []" + fi + for option_id in $( seq 1 $# ) ; do + local option="$( eval "echo \$$option_id")" + local key="$( echo $option | awk -F "=" '{print $1}' )"; + local value="$( echo $option | awk -F "=" '{print $2}' )"; + + #essentials: shortname, longname, description + if [ "$key" = "short" ]; then + local shortname="$value" + if [ ${#shortname} -ne 1 ]; then + optparse.throw_error "short name expected to be one character long" + fi + local short="-${shortname}" + elif [ "$key" = "long" ]; then + local longname="$value" + if [ ${#longname} -lt 2 ]; then + optparse.throw_error "long name expected to be atleast one character long" + fi + local long="--${longname}" + elif [ "$key" = "desc" ]; then + local desc="$value" + elif [ "$key" = "default" ]; then + local default="$value" + elif [ "$key" = "variable" ]; then + local variable="$value" + elif [ "$key" = "value" ]; then + local val="$value" + fi + done + + if [ "$variable" = "" ]; then + optparse.throw_error "You must give a variable for option: ($short/$long)" + fi + + if [ "$val" = "" ]; then + val="\$OPTARG" + fi + + # build OPTIONS and help + optparse_usage="${optparse_usage}#NL#TB${short} $(printf "%-25s %s" "${long}:" "${desc}")" + if [ "$default" != "" ]; then + optparse_usage="${optparse_usage} [default:$default]" + fi + optparse_contractions="${optparse_contractions}#NL#TB#TB${long})#NL#TB#TB#TBparams=\"\$params ${short}\";;" + if [ "$default" != "" ]; then + optparse_defaults="${optparse_defaults}#NL${variable}=${default}" + fi + optparse_arguments_string="${optparse_arguments_string}${shortname}" + if [ "$val" = "\$OPTARG" ]; then + optparse_arguments_string="${optparse_arguments_string}:" + fi + optparse_process="${optparse_process}#NL#TB#TB${shortname})#NL#TB#TB#TB${variable}=\"$val\";;" +} + +# ----------------------------------------------------------------------------------------------------------------------------- +function optparse.build(){ + local build_file="$(mktemp -t "optparse-XXXXXX.tmp")" + + # Building getopts header here + + # Function usage + cat << EOF > $build_file +function usage(){ +cat << XXX +usage: \$0 [OPTIONS] +OPTIONS: + $optparse_usage + -? --help : usage +XXX +} +# Contract long options into short options +params="" +while [ \$# -ne 0 ]; do + param="\$1" + shift + case "\$param" in + $optparse_contractions + "-?"|--help) + usage + exit 0;; + *) + if [[ "\$param" == --* ]]; then + echo -e "Unrecognized long option: \$param" + usage + exit 1 + fi + params="\$params \"\$param\"";; + esac +done +eval set -- "\$params" +# Set default variable values +$optparse_defaults +# Process using getopts +while getopts "$optparse_arguments_string" option; do + case \$option in + # Substitute actions for different variables + $optparse_process + :) + echo "Option - \$OPTARG requires an argument" + exit 1;; + *) + usage + exit 1;; + esac +done +# Clean up after self +rm $build_file +EOF + + local -A o=( ['#NL']='\n' ['#TB']='\t' ) + + for i in "${!o[@]}"; do + sed -i "s/${i}/${o[$i]}/g" $build_file + done + + # Unset global variables + unset optparse_usage + unset optparse_process + unset optparse_arguments_string + unset optparse_defaults + unset optparse_contractions + + # Return file name to parent + echo "$build_file" +} \ No newline at end of file