From 8ba4920b324a9d8f0781332dacf9a492ac35b6d8 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Thu, 3 Feb 2022 12:57:23 +0000 Subject: [PATCH] build dir clean-up and initial MCS bootstrap script version to simplify onboarding --- build/Build.bat | 82 ----------- build/CalpontVersion.txt.in | 13 -- build/CopyToCalweb.bat | 5 - build/Doxyfile | 275 ----------------------------------- build/Nightly.bat | 80 ---------- build/README | 75 ---------- build/README.md | 22 ++- build/Switch.bat | 36 ----- build/bootstrap_mcs.sh | 179 +++++++++++++++++++++++ build/build-boost-win.bat | 11 -- build/clean | 37 ----- build/columnstore_startup.sh | 15 -- build/docBuild | 6 - build/infinidb-build.sh | 64 -------- build/nightlyDiff.sh | 52 ------- build/randomSleep.sh | 9 -- 16 files changed, 188 insertions(+), 773 deletions(-) delete mode 100755 build/Build.bat delete mode 100644 build/CalpontVersion.txt.in delete mode 100644 build/CopyToCalweb.bat delete mode 100644 build/Doxyfile delete mode 100644 build/Nightly.bat delete mode 100644 build/README delete mode 100644 build/Switch.bat create mode 100755 build/bootstrap_mcs.sh delete mode 100644 build/build-boost-win.bat delete mode 100755 build/clean delete mode 100755 build/columnstore_startup.sh delete mode 100755 build/docBuild delete mode 100755 build/infinidb-build.sh delete mode 100755 build/nightlyDiff.sh delete mode 100755 build/randomSleep.sh diff --git a/build/Build.bat b/build/Build.bat deleted file mode 100755 index 0ee1bea39..000000000 --- a/build/Build.bat +++ /dev/null @@ -1,82 +0,0 @@ -::@echo off - -set branch=%1 -set basedir=\InfiniDB_%branch% -IF "%branch%" == "" ( - set branch=develop - set basedir=\InfiniDB -) -IF "%branch%" == "trunk" ( - set branch=develop - set basedir=\InfiniDB -) -echo %branch% - -echo building %basedir% - -echo ====================================== -echo updating mysql source -cd %basedir%\mysql -git checkout %branch% -git stash -git pull -git stash pop - -echo ====================================== -echo updating InfiniDB source -cd %basedir%\genii -git checkout %branch% -git pull origin_http %branch% -IF %ERRORLEVEL% NEQ 0 GOTO ERROR_HANDLER - -echo ====================================== -echo Building mysql -cd %basedir%\mysql -MSBuild /M:8 /t:rebuild /p:Configuration="Release" /p:Platform="x64" mysql.sln -IF %ERRORLEVEL% NEQ 0 GOTO ERROR_HANDLER - -echo ====================================== -echo Building InfiniDB -cd %basedir%\genii\build -MSBuild /M:8 /t:rebuild /p:Configuration="EnterpriseRelease" /p:Platform="x64" InfiniDB.sln -IF %ERRORLEVEL% NEQ 0 GOTO ERROR_HANDLER - -echo ====================================== -echo Building infindb-ent -cd %basedir%\infinidb-ent\build -call Build.bat -IF %ERRORLEVEL% NEQ 0 GOTO ERROR_HANDLER -cd %basedir%\genii\build - -echo ====================================== -echo Building CalpontVersion.txt -call BuildCalpontVersion.bat - -echo ====================================== -echo Building the Standard Installer -makensis ..\utils\winport\idb_64_standard.nsi -IF %ERRORLEVEL% NEQ 0 GOTO ERROR_HANDLER - -echo ====================================== -echo Building the Enterprise Installer -makensis ..\utils\winport\idb_64_enterprise.nsi -IF %ERRORLEVEL% NEQ 0 GOTO ERROR_HANDLER -echo. - -echo Build complete -GOTO QUIT - -:ERROR_HANDLER -echo. -echo ====================================== -echo Error occured. InfiniDB not built - -cd %basedir%\genii\build -exit /B 1 - -:QUIT -echo ====================================== -cd %basedir%\genii\build -echo compiled at %date% %time% -echo nightly build complete - diff --git a/build/CalpontVersion.txt.in b/build/CalpontVersion.txt.in deleted file mode 100644 index 161d08c0b..000000000 --- a/build/CalpontVersion.txt.in +++ /dev/null @@ -1,13 +0,0 @@ -Name : infinidb Relocations: (not relocatable) -Version : @@VERSION@@ Vendor: InfiniDB, Inc. -Release : @@RELEASE@@ Build Date: @@BUILDDATE@@ -Install Date: @@INSTALLDATE@@ Build Host: @@BUILDMACHINE@@.calpont.com -Group : Applications -Size : 750449506 License: Copyright (c) 2014 InfiniDB, Inc., all rights reserved -Signature : (none) -Packager : @@PACKAGER@@ -URL : http://www.infinidb.co/ -Summary : InfiniDB software for Windows - -InfiniDB binary files - diff --git a/build/CopyToCalweb.bat b/build/CopyToCalweb.bat deleted file mode 100644 index 058af7931..000000000 --- a/build/CopyToCalweb.bat +++ /dev/null @@ -1,5 +0,0 @@ -set dailydirname=genii-%DATE:~10,4%-%DATE:~4,2%-%DATE:~7,2% -xcopy /i /y \InfiniDB\genii\utils\winport\InfiniDB64.exe \\calweb\shared\\Iterations\Latest\packages\ -xcopy /i /y \InfiniDB\genii\utils\winport\InfiniDB64.exe \\calweb\shared\\Iterations\nightly\%dailydirname%\packages\ -xcopy /i /y \InfiniDB\genii\utils\winport\InfiniDB64-ent.exe \\calweb\shared\\Iterations\Latest\packages\ -xcopy /i /y \InfiniDB\genii\utils\winport\InfiniDB64-ent.exe \\calweb\shared\\Iterations\nightly\%dailydirname%\packages\ diff --git a/build/Doxyfile b/build/Doxyfile deleted file mode 100644 index db68c3d8b..000000000 --- a/build/Doxyfile +++ /dev/null @@ -1,275 +0,0 @@ -# Doxyfile 1.4.1-KDevelop - -#--------------------------------------------------------------------------- -# Project related configuration options -#--------------------------------------------------------------------------- -PROJECT_NAME = build.kdevelop -PROJECT_NUMBER = $VERSION$ -OUTPUT_DIRECTORY = -CREATE_SUBDIRS = NO -OUTPUT_LANGUAGE = English -USE_WINDOWS_ENCODING = NO -BRIEF_MEMBER_DESC = YES -REPEAT_BRIEF = YES -ABBREVIATE_BRIEF = "The $name class" \ - "The $name widget" \ - "The $name file" \ - is \ - provides \ - specifies \ - contains \ - represents \ - a \ - an \ - the -ALWAYS_DETAILED_SEC = NO -INLINE_INHERITED_MEMB = NO -FULL_PATH_NAMES = YES -STRIP_FROM_PATH = /home/dhill/InfiniDB_MariaDB/oam/install_scripts/ -STRIP_FROM_INC_PATH = -SHORT_NAMES = NO -JAVADOC_AUTOBRIEF = NO -MULTILINE_CPP_IS_BRIEF = NO -DETAILS_AT_TOP = NO -INHERIT_DOCS = YES -DISTRIBUTE_GROUP_DOC = NO -TAB_SIZE = 8 -ALIASES = -OPTIMIZE_OUTPUT_FOR_C = NO -OPTIMIZE_OUTPUT_JAVA = NO -SUBGROUPING = YES -#--------------------------------------------------------------------------- -# Build related configuration options -#--------------------------------------------------------------------------- -EXTRACT_ALL = NO -EXTRACT_PRIVATE = NO -EXTRACT_STATIC = NO -EXTRACT_LOCAL_CLASSES = YES -EXTRACT_LOCAL_METHODS = NO -HIDE_UNDOC_MEMBERS = NO -HIDE_UNDOC_CLASSES = NO -HIDE_FRIEND_COMPOUNDS = NO -HIDE_IN_BODY_DOCS = NO -INTERNAL_DOCS = NO -CASE_SENSE_NAMES = YES -HIDE_SCOPE_NAMES = NO -SHOW_INCLUDE_FILES = YES -INLINE_INFO = YES -SORT_MEMBER_DOCS = YES -SORT_BRIEF_DOCS = NO -SORT_BY_SCOPE_NAME = NO -GENERATE_TODOLIST = YES -GENERATE_TESTLIST = YES -GENERATE_BUGLIST = YES -GENERATE_DEPRECATEDLIST= YES -ENABLED_SECTIONS = -MAX_INITIALIZER_LINES = 30 -SHOW_USED_FILES = YES -SHOW_DIRECTORIES = YES -FILE_VERSION_FILTER = -#--------------------------------------------------------------------------- -# configuration options related to warning and progress messages -#--------------------------------------------------------------------------- -QUIET = NO -WARNINGS = YES -WARN_IF_UNDOCUMENTED = YES -WARN_IF_DOC_ERROR = YES -WARN_NO_PARAMDOC = NO -WARN_FORMAT = "$file:$line: $text" -WARN_LOGFILE = -#--------------------------------------------------------------------------- -# configuration options related to the input files -#--------------------------------------------------------------------------- -INPUT = /home/dhill/InfiniDB_MariaDB/build -FILE_PATTERNS = *.c \ - *.cc \ - *.cxx \ - *.cpp \ - *.c++ \ - *.java \ - *.ii \ - *.ixx \ - *.ipp \ - *.i++ \ - *.inl \ - *.h \ - *.hh \ - *.hxx \ - *.hpp \ - *.h++ \ - *.idl \ - *.odl \ - *.cs \ - *.php \ - *.php3 \ - *.inc \ - *.m \ - *.mm \ - *.dox \ - *.C \ - *.CC \ - *.C++ \ - *.II \ - *.I++ \ - *.H \ - *.HH \ - *.H++ \ - *.CS \ - *.PHP \ - *.PHP3 \ - *.M \ - *.MM \ - *.C \ - *.H \ - *.tlh \ - *.diff \ - *.patch \ - *.moc \ - *.xpm \ - *.dox -RECURSIVE = yes -EXCLUDE = -EXCLUDE_SYMLINKS = NO -EXCLUDE_PATTERNS = -EXAMPLE_PATH = -EXAMPLE_PATTERNS = * -EXAMPLE_RECURSIVE = NO -IMAGE_PATH = -INPUT_FILTER = -FILTER_PATTERNS = -FILTER_SOURCE_FILES = NO -#--------------------------------------------------------------------------- -# configuration options related to source browsing -#--------------------------------------------------------------------------- -SOURCE_BROWSER = NO -INLINE_SOURCES = NO -STRIP_CODE_COMMENTS = YES -REFERENCED_BY_RELATION = YES -REFERENCES_RELATION = YES -VERBATIM_HEADERS = YES -#--------------------------------------------------------------------------- -# configuration options related to the alphabetical class index -#--------------------------------------------------------------------------- -ALPHABETICAL_INDEX = NO -COLS_IN_ALPHA_INDEX = 5 -IGNORE_PREFIX = -#--------------------------------------------------------------------------- -# configuration options related to the HTML output -#--------------------------------------------------------------------------- -GENERATE_HTML = YES -HTML_OUTPUT = html -HTML_FILE_EXTENSION = .html -HTML_HEADER = -HTML_FOOTER = -HTML_STYLESHEET = -HTML_ALIGN_MEMBERS = YES -GENERATE_HTMLHELP = NO -CHM_FILE = -HHC_LOCATION = -GENERATE_CHI = NO -BINARY_TOC = NO -TOC_EXPAND = NO -DISABLE_INDEX = NO -ENUM_VALUES_PER_LINE = 4 -GENERATE_TREEVIEW = NO -TREEVIEW_WIDTH = 250 -#--------------------------------------------------------------------------- -# configuration options related to the LaTeX output -#--------------------------------------------------------------------------- -GENERATE_LATEX = YES -LATEX_OUTPUT = latex -LATEX_CMD_NAME = latex -MAKEINDEX_CMD_NAME = makeindex -COMPACT_LATEX = NO -PAPER_TYPE = a4wide -EXTRA_PACKAGES = -LATEX_HEADER = -PDF_HYPERLINKS = NO -USE_PDFLATEX = NO -LATEX_BATCHMODE = NO -LATEX_HIDE_INDICES = NO -#--------------------------------------------------------------------------- -# configuration options related to the RTF output -#--------------------------------------------------------------------------- -GENERATE_RTF = NO -RTF_OUTPUT = rtf -COMPACT_RTF = NO -RTF_HYPERLINKS = NO -RTF_STYLESHEET_FILE = -RTF_EXTENSIONS_FILE = -#--------------------------------------------------------------------------- -# configuration options related to the man page output -#--------------------------------------------------------------------------- -GENERATE_MAN = NO -MAN_OUTPUT = man -MAN_EXTENSION = .3 -MAN_LINKS = NO -#--------------------------------------------------------------------------- -# configuration options related to the XML output -#--------------------------------------------------------------------------- -GENERATE_XML = yes -XML_OUTPUT = xml -XML_SCHEMA = -XML_DTD = -XML_PROGRAMLISTING = YES -#--------------------------------------------------------------------------- -# configuration options for the AutoGen Definitions output -#--------------------------------------------------------------------------- -GENERATE_AUTOGEN_DEF = NO -#--------------------------------------------------------------------------- -# configuration options related to the Perl module output -#--------------------------------------------------------------------------- -GENERATE_PERLMOD = NO -PERLMOD_LATEX = NO -PERLMOD_PRETTY = YES -PERLMOD_MAKEVAR_PREFIX = -#--------------------------------------------------------------------------- -# Configuration options related to the preprocessor -#--------------------------------------------------------------------------- -ENABLE_PREPROCESSING = YES -MACRO_EXPANSION = NO -EXPAND_ONLY_PREDEF = NO -SEARCH_INCLUDES = YES -INCLUDE_PATH = -INCLUDE_FILE_PATTERNS = -PREDEFINED = -EXPAND_AS_DEFINED = -SKIP_FUNCTION_MACROS = YES -#--------------------------------------------------------------------------- -# Configuration::additions related to external references -#--------------------------------------------------------------------------- -TAGFILES = -GENERATE_TAGFILE = build.tag -ALLEXTERNALS = NO -EXTERNAL_GROUPS = YES -PERL_PATH = /usr/bin/perl -#--------------------------------------------------------------------------- -# Configuration options related to the dot tool -#--------------------------------------------------------------------------- -CLASS_DIAGRAMS = YES -HIDE_UNDOC_RELATIONS = YES -HAVE_DOT = NO -CLASS_GRAPH = YES -COLLABORATION_GRAPH = YES -GROUP_GRAPHS = YES -UML_LOOK = NO -TEMPLATE_RELATIONS = NO -INCLUDE_GRAPH = YES -INCLUDED_BY_GRAPH = YES -CALL_GRAPH = NO -GRAPHICAL_HIERARCHY = YES -DIRECTORY_GRAPH = YES -DOT_IMAGE_FORMAT = png -DOT_PATH = -DOTFILE_DIRS = -MAX_DOT_GRAPH_WIDTH = 1024 -MAX_DOT_GRAPH_HEIGHT = 1024 -MAX_DOT_GRAPH_DEPTH = 1000 -DOT_TRANSPARENT = NO -DOT_MULTI_TARGETS = NO -GENERATE_LEGEND = YES -DOT_CLEANUP = YES -#--------------------------------------------------------------------------- -# Configuration::additions related to the search engine -#--------------------------------------------------------------------------- -SEARCHENGINE = NO diff --git a/build/Nightly.bat b/build/Nightly.bat deleted file mode 100644 index 63014d639..000000000 --- a/build/Nightly.bat +++ /dev/null @@ -1,80 +0,0 @@ -:: This nightly script is customized for srvbldwin1 -:: It probably needs to be modified if used for other machines -:: Nightly.bat needs be moved to c:\ and run from there. - -::@echo off - -:: If there's a command line branch, use it. -set branch=%1 -IF "%1" == "" ( -:: No command line, use file - FOR /F %%i IN (branch.txt) DO set branch=%%i -) -echo %branch% -IF "%branch%" == "trunk" ( - set basedir=\InfiniDB -) ELSE ( - set basedir=\InfiniDB_%branch% -) - -:: Setup branch.txt for next run -:: default is to run trunk next -set nextrun=trunk -echo %nextrun% -IF "%branch%" == "trunk" set nextrun=4.0 -echo %nextrun% -IF "%branch%" == "4.0" set nextrun=4.5 -echo %nextrun% -echo %nextrun% > branch.txt - -echo Building %branch% at %basedir% - -:: Checkout the server -bash %basedir%\genii\tools\reserveStacks\stack reserve srvbldwin1 nightly f:/Calpont - -:: create a time with a leading zero if hour < 10 -set MYTIME=%TIME: =0% -set archivedirname=%DATE:~10,4%-%DATE:~4,2%-%DATE:~7,2%@%MYTIME:~0,2%.%MYTIME:~3,2%.%MYTIME:~6,2% - -echo Building the application -cd %basedir%\genii\build - -call Build.bat %branch% -IF %ERRORLEVEL% NEQ 0 GOTO ERROR_HANDLER - -echo Running the installer -cd %basedir%\genii\utils\winport -InfiniDB64-ent.exe /S /D=f:\Calpont -IF %ERRORLEVEL% NEQ 0 GOTO ERROR_HANDLER -cd \ - -echo Waiting for InfiniDB service to boot -call svcwait.bat -IF %ERRORLEVEL% NEQ 0 GOTO ERROR_HANDLER -echo InfiniDB service is ready - -echo copying the installer to calweb -call %basedir%\genii\build\CopyToCalweb.bat - -echo running the nightly test scripts -cd %basedir%\genii\mysql\queries\nightly\srvswdev11 -bash ./go.sh f:/Calpont -GOTO QUIT - -:ERROR_HANDLER -echo. -echo Build error occured. Nightly tests not run >> build.log -cd \ -expect %basedir%/genii/build/CopyLog.sh -xcopy build.log f:\nightly\build_log\%archivedirname%\ -bash %basedir%\genii\tools\reserveStacks\stack release srvbldwin1 nightly f:/Calpont -exit 1 - -:QUIT -echo nightly tests complete >> go.log -echo %date% %time% >> go.log -cd \ -expect %basedir%/genii/build/CopyLog.sh -xcopy build.log f:\nightly\build_log\%archivedirname%\ -bash %basedir%\genii\tools\reserveStacks\stack release srvbldwin1 nightly f:/Calpont -:END \ No newline at end of file diff --git a/build/README b/build/README deleted file mode 100644 index 2958b57a8..000000000 --- a/build/README +++ /dev/null @@ -1,75 +0,0 @@ -How it Works Now ----------------- - - -There is a new toplevel directory called build. This contains a -couple of scripts and a couple of previously (as far as I can tell) -unmanaged files: Columnstore.xml and Doxyfile. It also contains a -top-level Makefile and a rules.tmpl, which is a template rules file -for other makefiles to include. More on that later. - -After doing an updateGenii.pl, you do the following from the top of -your development tree, which, more or less has to be genii in -conformance with updateGenii.pl. (I'm not particularly happy with what -we are doing with updateGenii.pl, but, it is the prevailing custom and -these changes assume its use). - -0. cd $HOME/genii. - -1. build/bootstrap - - This does the following: - - a. creates export/bin,lib,share,etc,include - - b. Copies static elements into the export tree. Static elements - are things we use but either don't build or don't build - routinely. This includes Doxyfile, Columnstore.xml and - net-snmp components. - - c. Generates a rules.mak from build/build/rules.tmpl to pwd (which - is supposed to be the top of your development tree). rules.mak - includes shared variables used by many other makefiles, mostly - in install rules. In particular, it defines TOP, which is how - build processes find the export tree. Makefiles that use this - have a hard-coded path to rules.mak. These hard-coded paths can - be automatically reset via build/setmktop. - - d. Runs make bootstrap for components that need to export headers. - bootstrap rules usually just copy LINCLUDES (headers that need - to be shared with other components) to export/include. - -2. Now you can type make in the various component areas or use the - top-level Makefile installed by bootstrap. - - The order specified in the buildbot configuration is: - - a. utils - b. snmpd - c. oam - d. dbcon --------------- - e. emulator - f. mysql - g. exemgr - h. procmon - i. procmgr - -3. There is a top-level build/Makefile to support development build - activities. It is not part of the formal nightly build. The - buildbot setup orchestrates component builds via component specific - builders. Developers may want to use this. cp build/Makefile - $HOME/genii or use make -f. - - For example you can try make scratch to build everything including - test, coverage, etc., or make compile to just try the build part. - It's a simple makefile, you can read it if you want to. - -4. Note that while we no longer install files into /usr/local/Calpont, - the old install targets are retained as the mechanism for copying - programs and libraries into the export tree. Install targets do - not operate on header files. This is taken care of by the - bootstrap target. - -5. Following a build, a bunch of output products should be in - genii/export. diff --git a/build/README.md b/build/README.md index 39cde32aa..9ee3599bc 100644 --- a/build/README.md +++ b/build/README.md @@ -1,20 +1,16 @@ -This is InfiniDB 4.0. +This is MariaDB Columnstore ===================== -To build InfiniDB from source you will need: +To build MCS from source you will need: - * a CentOS/RHEL 5/6, debian 5/6 or Ubuntu 10/12 linux host configured for software development - -Along with a working C++ compiler and GNU software development tools you will need the following extra packages: - - * expect - * zlib-devel - * ncurses-devel - * libxml2-devel + * modern linux distribution. e.g. Ubuntu 20 Clone or download this repository. - git clone https://github.com/infinidb/infinidb - cd infinidb - ./build/src-build --prefix=$HOME + git clone https://github.com/MariaDB/server +Edit a bootstrap script to fix paths to MariaDB repo cloned. + vim /some/path/server/storage/columnstore/columnstore/build/bootstrap_mcs.sh + +Run the bootstrap + /some/path/server/storage/columnstore/columnstore/build/bootstrap_mcs.sh diff --git a/build/Switch.bat b/build/Switch.bat deleted file mode 100644 index 9bd4e6e2a..000000000 --- a/build/Switch.bat +++ /dev/null @@ -1,36 +0,0 @@ -::@echo off -IF NOT EXIST C:\InfiniDB_3.6\NUL GOTO SWITCH_to_trunk -IF NOT EXIST C:\InfiniDB_4.0\NUL GOTO SWITCH_to_3.6 -IF NOT EXIST C:\InfiniDB_trunk\NUL GOTO SWITCH_to_4.0 - -:SWITCH_to_trunk -echo switching to trunk -REN "\InfiniDB" "InfiniDB_3.6" -IF %ERRORLEVEL% NEQ 0 GOTO ERROR_HANDLER -REN "\InfiniDB_trunk" "InfiniDB" -IF %ERRORLEVEL% NEQ 0 GOTO ERROR_HANDLER -echo Switched to InfiniDB trunk -GOTO COMPLETE - -:SWITCH_to_3.6 -echo switching to 3.6 -REN "\InfiniDB" "InfiniDB_4.0" -IF %ERRORLEVEL% NEQ 0 GOTO ERROR_HANDLER -REN "\InfiniDB_3.6" "InfiniDB" -IF %ERRORLEVEL% NEQ 0 GOTO ERROR_HANDLER -echo Switched to InfiniDB 3.6 -GOTO COMPLETE - -:SWITCH_to_4.0 -echo switching to 4.0 -REN "\InfiniDB" "InfiniDB_trunk" -IF %ERRORLEVEL% NEQ 0 GOTO ERROR_HANDLER -REN "\InfiniDB_4.0" "InfiniDB" -IF %ERRORLEVEL% NEQ 0 GOTO ERROR_HANDLER -echo Switched to InfiniDB 4.0 -GOTO COMPLETE - -:ERROR_HANDLER -echo failed to switch - -:COMPLETE diff --git a/build/bootstrap_mcs.sh b/build/bootstrap_mcs.sh new file mode 100755 index 000000000..8187e5e0c --- /dev/null +++ b/build/bootstrap_mcs.sh @@ -0,0 +1,179 @@ +#!/bin/sh + +# This script compiles/installs MCS from scratch and it makes some assumptions: +# - the server's source code is two directories above the MCS engine source. +# - the script is to be run under root. +# - it is run MDB_MDB_SOURCE_PATH/storage/columnstore/columnstore, e.g. ./build/bootstrap_mcs.sh + +DISTRO=$1 +MDB_BUILD_TYPE=$2 +MDB_SOURCE_PATH=`pwd`/../../../ +MCS_CONFIG_DIR=/etc/columnstore +# Needs systemd to be installed obviously. +# Feel free to ask in MariaDB Zulip how to bootstrap MCS in containers or other systemd-free environments. +systemctl stop mariadb-columnstore +MDB_GIT_URL=https://github.com/MariaDB/server.git +MDB_GIT_TAG=10.8 + +if [ -z "$DISTRO" ]; then + echo "Choose a distro" + exit 1 +fi + +if [ $DISTRO = 'bionic' ]; then + sudo apt-get -y update + apt-get -y install build-essential automake libboost-all-dev bison cmake \ + libncurses5-dev libaio-dev libsystemd-dev libpcre2-dev \ + libperl-dev libssl-dev libxml2-dev libkrb5-dev flex libpam-dev git \ + libsnappy-dev libcurl4-openssl-dev +elif [ $DISTRO = 'focal' ]; then + sudo apt-get -y update + apt-get -y install build-essential automake libboost-all-dev bison cmake \ + libncurses5-dev libaio-dev libsystemd-dev libpcre2-dev \ + libperl-dev libssl-dev libxml2-dev libkrb5-dev flex libpam-dev git \ + libsnappy-dev libcurl4-openssl-dev +elif [ $DISTRO = 'centos' ]; then + yum -y install epel-release \ + && yum -y groupinstall "Development Tools" \ + && yum -y install bison ncurses-devel readline-devel perl-devel openssl-devel cmake libxml2-devel gperf libaio-devel libevent-devel python-devel ruby-devel tree wget pam-devel snappy-devel libicu \ + && yum -y install vim wget strace ltrace gdb rsyslog net-tools openssh-server expect boost perl-DBI libicu boost-devel initscripts jemalloc-devel libcurl-devel +elif [ $DISTRO = 'leap' ]; then + zypper install -y bison ncurses-devel readline-devel libopenssl-devel cmake libxml2-devel gperf libaio-devel libevent-devel python-devel ruby-devel tree wget pam-devel snappy-devel libicu-devel \ + && zypper install -y libboost_system-devel libboost_filesystem-devel libboost_thread-devel libboost_regex-devel libboost_date_time-devel libboost_chrono-devel \ + && zypper install -y vim wget strace ltrace gdb rsyslog net-tools expect perl-DBI libicu boost-devel jemalloc-devel libcurl-devel \ + && zypper install -y gcc gcc-c++ git automake libtool +fi + +if [ ! -d $MDB_SOURCE_PATH ]; then + git clone $MDB_GIT_URL $MDB_SOURCE_PATH -b $MDB_GIT_TAG +fi + +if [ ! -d $MCS_CONFIG_DIR ]; then + mkdir $MCS_CONFIG_DIR +fi + +if [ -z "$(grep mysql /etc/passwd)" ]; then + echo "Adding user mysql into /etc/passwd" + useradd -r -U mysql -d /var/lib/mysql + exit 1 +fi + +if [ -z "$(grep mysql /etc/group)" ]; then + echo "You need to manually add mysql group into /etc/group, e.g. mysql:x:999" + exit 1 +fi + +MCS_INSTALL_PREFIX=/var/lib/ +rm -rf /var/lib/columnstore/data1/* +rm -rf /var/lib/columnstore/data/ +rm -rf /var/lib/columnstore/local/ +rm -f /var/lib/columnstore/storagemanager/storagemanager-lock +rm -f /var/lib/columnstore/storagemanager/cs-initialized + +MCS_TMP_DIR=/tmp/columnstore_tmp_files +TMP_PATH=/tmp +CPUS=$(getconf _NPROCESSORS_ONLN) + +# script +rm -rf $MCS_TMP_DIR/* +rm -rf /var/lib/mysql + +cd $MDB_SOURCE_PATH +if [[ "$DISTRO" = 'bionic' || "$DISTRO" = 'focal' ]]; then + #MDB_CMAKE_FLAGS='-DWITH_SYSTEMD=yes -DPLUGIN_TOKUDB=NO -DPLUGIN_ROCKSDB=NO -DPLUGIN_MROONGA=NO -DPLUGIN_GSSAPI=NO -DWITH_MARIABACKUP=NO -DDEB=bionic -DPLUGIN_COLUMNSTORE=YES' + MDB_CMAKE_FLAGS='-DWITH_SYSTEMD=yes -DPLUGIN_COLUMNSTORE=YES -DPLUGIN_MROONGA=NO -DPLUGIN_ROCKSDB=NO -DPLUGIN_TOKUDB=NO -DPLUGIN_CONNECT=NO -DPLUGIN_SPIDER=NO -DPLUGIN_OQGRAPH=NO -DPLUGIN_SPHINX=NO -DBUILD_CONFIG=mysql_release -DWITH_WSREP=OFF -DWITH_SSL=system -DDEB=bionic' + # Some development flags + #MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_GTEST=1 -DWITH_ROWGROUP_UT=1 -DWITH_DATACONVERT_UT=1 -DWITH_ARITHMETICOPERATOR_UT=1 -DWITH_ORDERBY_UT=1 -DWITH_CSDECIMAL_UT=1 -DWITH_SORTING_COMPARATORS_UT=1" + MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_MICROBENCHMARKS=YES -DWITH_BRM_UT=YES" #-DWITH_GTEST=1 -DWITH_UNITTESTS=YES + cmake . -DCMAKE_BUILD_TYPE=$MDB_BUILD_TYPE ${MDB_CMAKE_FLAGS} && \ + make -j $CPUS install +elif [ $DISTRO = 'centos' ]; then + MDB_CMAKE_FLAGS='-DWITH_SYSTEMD=yes -DPLUGIN_AUTH_GSSAPI=NO -DPLUGIN_COLUMNSTORE=YES -DPLUGIN_MROONGA=NO -DPLUGIN_ROCKSDB=NO -DPLUGIN_TOKUDB=NO -DPLUGIN_CONNECT=NO -DPLUGIN_SPIDER=NO -DPLUGIN_OQGRAPH=NO -DPLUGIN_SPHINX=NO -DBUILD_CONFIG=mysql_release -DWITH_WSREP=OFF -DWITH_SSL=system -DRPM=CentOS7' + cmake . -DCMAKE_BUILD_TYPE=$MDB_BUILD_TYPE $MDB_CMAKE_FLAGS && \ + make -j $CPUS install +fi + +if [ $? -ne 0 ]; then + return 1 +fi + +# These two lines are to handle file layout difference b/w RPM- and DEB-based distributions. +# One of the lines always fails. +mv /usr/lib/mysql/plugin/ha_columnstore.so /tmp/ha_columnstore_1.so +mv /usr/lib64/mysql/plugin/ha_columnstore.so /tmp/ha_columnstore_2.so + + +/usr/bin/mysql_install_db --rpm --user=mysql + +mv /tmp/ha_columnstore_1.so /usr/lib/mysql/plugin/ha_columnstore.so +mv /tmp/ha_columnstore_2.so /usr/lib64/mysql/plugin/ha_columnstore.so + +cp -r /etc/mysql/conf.d /etc/my.cnf.d/ +cp $MDB_SOURCE_PATH/storage/columnstore/columnstore/oam/etc/Columnstore.xml /etc/columnstore/Columnstore.xml +cp $MDB_SOURCE_PATH/storage/columnstore/oam/etc/Columnstore.xml /etc/columnstore/Columnstore.xml +cp $MDB_SOURCE_PATH/storage/columnstore/columnstore/storage-manager/storagemanager.cnf /etc/columnstore/storagemanager.cnf +cp $MDB_SOURCE_PATH/storage/columnstore/storage-manager/storagemanager.cnf /etc/columnstore/storagemanager.cnf + +if [[ "$DISTRO" = 'bionic' || "$DISTRO" = 'focal' ]]; then + cp ./support-files/*.service /lib/systemd/system/ + cp ./storage/columnstore/columnstore/oam/install_scripts/*.service /lib/systemd/system/ + cp ./storage/columnstore/oam/install_scripts/*.service /lib/systemd/system/ + cp ./debian/additions/debian-start.inc.sh /usr/share/mysql/debian-start.inc.sh + cp ./debian/additions/debian-start /etc/mysql/debian-start + systemctl daemon-reload + rm -f /etc/mysql/my.cnf + cp -r /etc/mysql/conf.d/ /etc/my.cnf.d + cp -rp /etc/mysql/mariadb.conf.d/ /etc/my.cnf.d + mkdir /var/lib/columnstore/data1 + mkdir /var/lib/columnstore/data1/systemFiles + mkdir /var/lib/columnstore/data1/systemFiles/dbrm + chown -R mysql:mysql /var/lib/mysql + chown -R mysql:mysql /var/lib/mysql + chown -R mysql.mysql /var/run/mysqld + chown -R mysql:mysql /data/columnstore/* + chmod +x /usr/bin/mariadb* + cp /etc/my.cnf.d/mariadb.conf.d/columnstore.cnf /etc/my.cnf.d/ + + ldconfig + columnstore-post-install + chown -R mysql:mysql /data/columnstore/* + + /usr/sbin/install_mcs_mysql.sh + + chown -R syslog.syslog /var/log/mariadb/ + chmod 777 /var/log/mariadb/ + chmod 777 /var/log/mariadb/columnstore + +elif [ $DISTRO = 'centos' ]; then + cp ./support-files/*.service /lib/systemd/system/ + cp ./storage/columnstore/columnstore/oam/install_scripts/*.service /lib/systemd/system/ + cp ./storage/columnstore/oam/install_scripts/*.service /lib/systemd/system/ + systemctl daemon-reload + rm -f /etc/mysql/my.cnf + cp -r /etc/mysql/conf.d/ /etc/my.cnf.d + mkdir /var/lib/columnstore/data1 + mkdir /var/lib/columnstore/data1/systemFiles + mkdir /var/lib/columnstore/data1/systemFiles/dbrm + chown -R mysql:mysql /var/lib/mysql + chown -R mysql:mysql /data/columnstore/* + chown -R mysql.mysql /var/run/mysqld + chmod +x /usr/bin/mariadb* + + ldconfig + columnstore-post-install + chown -R mysql:mysql /data/columnstore/* + + /usr/sbin/install_mcs_mysql.sh + + /usr/sbin/install_mcs_mysql.sh + mkdir /var/lib/columnstore/data1 + mkdir /var/lib/columnstore/data1/systemFiles + mkdir /var/lib/columnstore/data1/systemFiles/dbrm + + chown -R mysql:mysql /var/log/mariadb/ + chmod 777 /var/log/mariadb/ + chmod 777 /var/log/mariadb/columnstore +fi + + +exit 0 diff --git a/build/build-boost-win.bat b/build/build-boost-win.bat deleted file mode 100644 index e4620edc1..000000000 --- a/build/build-boost-win.bat +++ /dev/null @@ -1,11 +0,0 @@ -REM You need to uncomment one of these sections below at a time -REM Then you need to copy the libs from stage to \InfiniDB -REM -REM build just 64-bit release libs -REM b2 --clean -b2 --with-date_time --with-filesystem --with-regex --with-system --with-thread --with-chrono address-model=64 variant=release stage - -REM build just 64-bit debug libs -REM bjam --clean -REM bjam --with-date_time --with-filesystem --with-regex --with-system --with-thread address-model=64 variant=debug stage - diff --git a/build/clean b/build/clean deleted file mode 100755 index d85e3b61a..000000000 --- a/build/clean +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/sh -# -# $Id: clean 1843 2013-03-14 21:43:13Z pleblanc $ -# - -for x in \ -dbcon \ -ddlproc \ -decomsvr \ -dmlproc \ -exemgr \ -mysql \ -net-snmp \ -oam \ -oamapps \ -primitives \ -procmgr \ -procmon \ -snmpd \ -tools \ -utils \ -versioning \ -writeengine \ -; do - -# BOOTSTRAP is used to have make ignore sections of make code like: -# -include $(SRCS:.cpp=.d) tdriver.d. Make will try to generate the .d -# files if they don't exist since we typically have a rule that says -# how to do that. That involves processing header files and we don't -# want the success of the clean operation to fail because of missing -# headers. - - make BOOTSTRAP=1 -C $x clean -done - -rm -rf export - diff --git a/build/columnstore_startup.sh b/build/columnstore_startup.sh deleted file mode 100755 index da9f2dede..000000000 --- a/build/columnstore_startup.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -# -# $Id: columnstore_startup.sh 3705 2013-08-07 19:47:20Z dhill $ -# -# columnstore_startup.sh steps for columnstore single server startup after install - -quiet=0 - -columnstore-post-install -echo -e "1\n1\n1\n1\n1\n1\n" | postConfigure -if [ $? -eq 0 ]; then - exit 0 -else - exit 1 -fi diff --git a/build/docBuild b/build/docBuild deleted file mode 100755 index 63efec47a..000000000 --- a/build/docBuild +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/sh - -if [ -f buildFlags ]; then - echo "Build Flags: `cat buildFlags`" -fi - diff --git a/build/infinidb-build.sh b/build/infinidb-build.sh deleted file mode 100755 index 31b9c7481..000000000 --- a/build/infinidb-build.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/bash - -infinidbBranch=$1 -mysqlBranch=$2 - -if [ -z "$infinidbBranch" ]; then - echo Usage: $0 infinidb-branch mysql-branch 1>&2 - exit 1 -fi - -if [ "$infinidbBranch" != "current" ]; then - if [ -z "$mysqlBranch" ]; then - echo Usage: $0 infinidb-branch mysql-branch 1>&2 - exit 1 - fi -else - mysqlBranch=$1 -fi - -{ -if [ "$infinidbBranch" != "current" ]; then - rm -rf InfiniDB-MySQL - rm -rf InfiniDB - rm -rf InfiniDB-Enterprise - - git clone https://github.com/mariadb-corporation/InfiniDB - git clone https://github.com/mariadb-corporation/InfiniDB-MySQL - git clone https://github.com/mariadb-corporation/InfiniDB-Enterprise -fi - -echo "****** Compile InfiniDB-MySQL" -cd InfiniDB-MySQL -if [ "$infinidbBranch" != "current" ]; then - git checkout $mysqlBranch -fi -./build-MySQL $mysqlBranch - -echo "****** Compile InfiniDB" -cd ../InfiniDB -if [ "$infinidbBranch" != "current" ]; then - git checkout $infinidbBranch -fi -./build/bootstrap -make -j4 -make -j4 -make install - -echo "****** Compile InfiniDB-Enterprise" -cd ../InfiniDB-Enterprise -if [ "$infinidbBranch" != "current" ]; then - git checkout $infinidbBranch -fi -make -j4 -make -j4 -make install - -echo "****** Generate RPMS" -cd ../InfiniDB -./build/build_rpms -} > infinidb-build-$infinidbBranch.log 2>&1 - -echo "InfiniDB Build Completed for $infinidbBranch" -exit 0 - diff --git a/build/nightlyDiff.sh b/build/nightlyDiff.sh deleted file mode 100755 index 0e3c1c5ac..000000000 --- a/build/nightlyDiff.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/bash -# $Id$ -# -# displays the changes in git develop for yesterday - -usage() -{ - echo "usage: nightlyDiff.sh [-sfh] [-d days] [-b branch]" - echo " -h display this help" - echo " -s display only a summary of the files changed" - echo " -f display fancy diffstat" - echo " -d days go back days days" -# echo " -b branch display changes for branch branch instead of develop" -} - -urlencode() -{ - local arg - arg="$1" - while [[ "$arg" =~ ^([0-9a-zA-Z/:_\.\-]*)([^0-9a-zA-Z/:_\.\-])(.*) ]] ; do - echo -n "${BASH_REMATCH[1]}" - printf "%%%X" "'${BASH_REMATCH[2]}'" - arg="${BASH_REMATCH[3]}" - done - echo -n "$arg" -} - -DAYSAGO=1 -BRANCH=develop -while getopts ":sfhd:b:" options; do - case $options in - s ) SUMMARY_ONLY="--name-only" ;; - f ) FANCY="--stat" ;; - d ) DAYSAGO=$OPTARG ;; - b ) BRANCH=$OPTARG ; echo "-b doesn't work yet" 1>&2; exit 1 ;; - h ) usage; exit 0 ;; - * ) usage; exit 1 ;; - esac -done - -secs=$(($(date +%s)-60*60*24*DAYSAGO)) - -dt1=$(date --date=@$secs +%Y-%m-%d) - -BRANCH=$(urlencode "$BRANCH") - -if [ $BRANCH != develop ]; then - BRANCH=branches/$BRANCH -fi - -git diff $SUMMARY_ONLY $FANCY "$BRANCH@{${dt1} 00:00:00}..$BRANCH@{${dt1} 23:59:59}" - diff --git a/build/randomSleep.sh b/build/randomSleep.sh deleted file mode 100755 index 5475e2d43..000000000 --- a/build/randomSleep.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -# This a tool for buildbot to keep the nightlies from swamping svn - -sleep=$((($RANDOM % 16) * 45)) - -echo sleep $sleep -sleep $sleep -