database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-30 19:23:07 +03:00
Code Activity

MCOL-3551 Use generic MariaDB Server paths

Browse Source 
This branch enforces the use of generic MariaDB server paths for their
binaries and data rather than custom paths.

/usr/local/mariadb/columnstore is now only for columnstore with this
patch.

It should be noted that this removes the auto-mounting of external
MariaDB UM data storage for AWS.

This is also a fix for MCOL-3510 after buildbot changes are made.

Also... MCOL-3552 Use columnstore.cnf to load plugins

The ColumnStore plugins now load using a columnstore.cnf instead of a
SQL sequence to be more in-line with MariaDB's methods.
This commit is contained in:
Andrew Hutchings
2019-10-10 14:28:02 +01:00
parent 5fd0873215
commit ebb22a96a3
58 changed files with 145 additions and 2143 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
oam/install_scripts/CMakeLists.txt
View File

@ -27,7 +27,6 @@ install(PROGRAMS post-install
columnstoreSyslog-ng
syslogSetup.sh
remote_scp_put.sh
columnstore.def
remotessh.exp
rsync.sh
remote_command_verify.sh

1
oam/install_scripts/columnstore.conf
View File

@ -1,2 +1 @@
/usr/local/mariadb/columnstore/lib
/usr/local/mariadb/columnstore/mysql/lib

9
oam/install_scripts/columnstore.def
View File

@ -1,9 +0,0 @@
#!/bin/bash
# Copy this file to /etc/default and rename it to 'columnstore'.
# Change this line to your MariaDB Columnstore installation directory
COLUMNSTORE_INSTALL_DIR=/usr/local/mariadb/columnstore
LD_LIBRARY_PATH=$COLUMNSTORE_INSTALL_DIR/lib:$COLUMNSTORE_INSTALL_DIR/mysql/lib
export COLUMNSTORE_INSTALL_DIR=$COLUMNSTORE_INSTALL_DIR
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH

4
oam/install_scripts/columnstore.in
View File

@ -51,8 +51,8 @@ export COLUMNSTORE_INSTALL_DIR=$COLUMNSTORE_INSTALL_DIR
InstallDir=$COLUMNSTORE_INSTALL_DIR
if [ $InstallDir != "/usr/local/mariadb/columnstore" ]; then
export PATH=$InstallDir/bin:$InstallDir/mysql/bin:/bin:/usr/bin
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$InstallDir/lib:$InstallDir/mysql/lib
export PATH=$InstallDir/bin:/bin:/usr/bin
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$InstallDir/lib
fi
#hadoop

2
oam/install_scripts/columnstoreAlias
View File

@ -1,6 +1,6 @@
# MariaDB Columnstore Alias Commands
#
alias mcsmysql='/usr/local/mariadb/columnstore/mysql/bin/mysql -u root'
alias mcsmysql='mysql -u root'
alias ma=/usr/local/mariadb/columnstore/bin/mcsadmin
alias mcsadmin=/usr/local/mariadb/columnstore/bin/mcsadmin
alias cpimport=/usr/local/mariadb/columnstore/bin/cpimport

9
oam/install_scripts/columnstoreLogRotate.in
View File

@ -18,12 +18,3 @@
copy
olddir @ENGINE_SYSCONFDIR@/columnstore
}
/usr/local/mariadb/columnstore/mysql/db/*.err {
missingok
rotate 7
daily
dateext
copytruncate
olddir /usr/local/mariadb/columnstore/mysql/db
su root root
}

6
oam/install_scripts/disable-rep-columnstore.sh
View File

@ -48,8 +48,7 @@ stop slave;
EOD
cat ${tmpdir}/idb_disable-rep.sql >>${tmpdir}/disable-rep-status.log
$installdir/mysql/bin/mysql \
--defaults-extra-file=$installdir/mysql/my.cnf \
mysql \
--user=root $pwprompt \
calpontsys <${tmpdir}/idb_disable-rep.sql >>${tmpdir}/disable-rep-status.log 2>&1
@ -64,8 +63,7 @@ reset slave;
EOD
cat ${tmpdir}/idb_disable-rep.sql >>${tmpdir}/disable-rep-status.log
$installdir/mysql/bin/mysql \
--defaults-extra-file=$installdir/mysql/my.cnf \
mysql \
--user=root $pwprompt \
calpontsys <${tmpdir}/idb_disable-rep.sql >>${tmpdir}/disable-rep-status.log 2>&1

2
oam/install_scripts/mariadb-command-line.sh
View File

@ -56,7 +56,7 @@ $command;
EOD
cat${tmpdir}/mariadb-command-line.sql >> ${tmpdir}/mariadb-command-line.log
$installdir/mysql/bin/mysql \
mysql \
--user=root \
calpontsys < ${tmpdir}/mariadb-command-line.sql >> ${tmpdir}/mariadb-command-line.log 2>&1

8
oam/install_scripts/master-rep-columnstore.sh
View File

@ -51,7 +51,7 @@ GRANT REPLICATION SLAVE ON *.* TO '$repUser'@'$hostipaddr';
EOD
cat ${tmpdir}/idb_master-rep.sql >>${tmpdir}/master-rep-status-$hostipaddr.log
$installdir/mysql/bin/mysql \
mysql \
--user=root \
calpontsys <${tmpdir}/idb_master-rep.sql >>${tmpdir}/master-rep-status-$hostipaddr.log 2>&1
@ -68,7 +68,7 @@ grant REPLICATION SLAVE on *.* to '$repUser'@'$hostipaddr' identified by 'Calpon
EOD
cat ${tmpdir}/idb_master-rep.sql >>${tmpdir}/master-rep-status-$hostipaddr.log
$installdir/mysql/bin/mysql \
mysql \
--user=root \
calpontsys <${tmpdir}/idb_master-rep.sql >>${tmpdir}/master-rep-status-$hostipaddr.log 2>&1
@ -83,7 +83,7 @@ SHOW MASTER STATUS
EOD
cat ${tmpdir}/idb_master-rep.sql >>${tmpdir}/master-rep-status-$hostipaddr.log
$installdir/mysql/bin/mysql \
mysql \
--user=root \
calpontsys <${tmpdir}/idb_master-rep.sql >>${tmpdir}/master-rep-status-$hostipaddr.log 2>&1
@ -95,7 +95,7 @@ SHOW MASTER STATUS
EOD
cat ${tmpdir}/idb_master-rep.sql >${tmpdir}/show-master-status.log
$installdir/mysql/bin/mysql \
mysql \
--user=root \
calpontsys <${tmpdir}/idb_master-rep.sql >>${tmpdir}/show-master-status.log

11
oam/install_scripts/module_installer.sh
View File

@ -50,8 +50,8 @@ shift $shiftcnt
if [ $installdir != "/usr/local/mariadb/columnstore" ]; then
export COLUMNSTORE_INSTALL_DIR=$installdir
export PATH=$COLUMNSTORE_INSTALL_DIR/bin:$COLUMNSTORE_INSTALL_DIR/mysql/bin:/bin:/usr/bin
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$COLUMNSTORE_INSTALL_DIR/lib:$COLUMNSTORE_INSTALL_DIR/mysql/lib
export PATH=$COLUMNSTORE_INSTALL_DIR/bin:/bin:/usr/bin
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$COLUMNSTORE_INSTALL_DIR/lib
else
export COLUMNSTORE_INSTALL_DIR=$installdir
fi
@ -93,13 +93,6 @@ if [ $cloud = "amazon-ec2" ] || [ $cloud = "amazon-vpc" ]; then
systemtype=`$COLUMNSTORE_INSTALL_DIR/bin/getConfig Installation ServerTypeInstall`
if [ $systemtype = "1" ]; then
umstoragetype=`$COLUMNSTORE_INSTALL_DIR/bin/getConfig Installation UMStorageType`
if [ $umstoragetype = "external" ]; then
echo "Setup UM Volume Mount"
device=`$COLUMNSTORE_INSTALL_DIR/bin/getConfig Installation UMVolumeDeviceName$mid`
mkdir -p $COLUMNSTORE_INSTALL_DIR/mysql/db > /dev/null 2>&1
sudo mount $device $COLUMNSTORE_INSTALL_DIR/mysql/db -t ext2 -o noatime,nodiratime,noauto,user
sudo chown $user:$user -R $COLUMNSTORE_INSTALL_DIR/mysql > /dev/null 2>&1
fi
fi
fi

10
oam/install_scripts/post-install.in
View File

@ -22,7 +22,7 @@ for arg in "$@"; do
if [ $installdirTmp == $installdir ] ; then
user=root
export COLUMNSTORE_INSTALL_DIR=$installdir
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$installdir/lib:$installdir/mysql/lib
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$installdir/lib
else
installdir=$installdirTmp
prefix=`dirname $installdir`
@ -43,7 +43,7 @@ done
if [ $user != "root" ]; then
export COLUMNSTORE_INSTALL_DIR=$installdir
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$installdir/lib:$installdir/mysql/lib
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$installdir/lib
else
# add library config file
/bin/cp -f $installdir/bin/columnstore.conf /etc/ld.so.conf.d/. >/dev/null 2>&1
@ -130,7 +130,7 @@ if [ $user != "root" ]; then
echo " " >> ${profileFile}
echo "# MariaDB Columnstore Non-Root Environment Variables" >> ${profileFile}
echo "export COLUMNSTORE_INSTALL_DIR=$COLUMNSTORE_INSTALL_DIR" >> ${profileFile}
echo "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$COLUMNSTORE_INSTALL_DIR/lib:$COLUMNSTORE_INSTALL_DIR/mysql/lib" >> ${profileFile}
echo "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$COLUMNSTORE_INSTALL_DIR/lib" >> ${profileFile}
fi
egrep -qs 'MariaDB Columnstore Non-Root Alias Variables' ${profileFile}
@ -207,10 +207,6 @@ mkdir -p $hdfsDir >/dev/null 2>&1
#create mount directories
mkdir /mnt/tmp > /dev/null 2>&1
# remove mysql archive log
test -d $installdir/mysql/db || mkdir -p $installdir/mysql/db
rm -rf $installdir/mysql/db/columnstore_log_archive > /dev/null 2>&1
# delete Columnstore shared memory segments
$installdir/bin/clearShm > /dev/null 2>&1

26
oam/install_scripts/post-mysql-install
View File

@ -22,9 +22,9 @@ checkForError() {
# See if engine columnstore exist
#---------------------------------------------------------------------------
echo "checking for engine columnstore..."
$installdir/mysql/bin/mysql \
mysql \
--user=root \
--execute='select * from mysql.plugin where name="columnstore";' \
--execute='show engines;' \
calpontsys | grep -i columnstore
#
@ -65,31 +65,11 @@ USER=`whoami 2>/dev/null`
if [ $USER != "root" ]; then
ldconfig >/dev/null 2>&1
export COLUMNSTORE_INSTALL_DIR=$installdir
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$COLUMNSTORE_INSTALL_DIR/lib:$COLUMNSTORE_INSTALL_DIR/mysql/lib
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$COLUMNSTORE_INSTALL_DIR/lib
else
ldconfig
fi
if [ -f $installdir/lib/libcalmysql.so.1.0.0 ]; then
libcalmysql=$installdir/lib/libcalmysql.so.1.0.0
libudfsdk=$installdir/lib/libudf_mysql.so.1.0.0
libregrsdk=$installdir/lib/libregr_mysql.so.1.0.0
elif [ -f $installdir/lib/libcalmysql.so.1 ]; then
libcalmysql=$installdir/lib/libcalmysql.so.1
libudfsdk=$installdir/lib/libudf_mysql.so.1
libregrsdk=$installdir/lib/libregr_mysql.so.1
else
libcalmysql=
fi
if [ -d $installdir/mysql/lib64/mysql/plugin -a -n "$libcalmysql" ]; then
cd $installdir/mysql/lib64/mysql/plugin
ln -sf $libcalmysql libcalmysql.so
ln -sf $libcalmysql libcalmysqlent.so
ln -sf $libudfsdk libudf_mysql.so
ln -sf $libregrsdk libregr_mysql.so
fi
if [ $installdir != "/usr/local/mariadb/columnstore" ]; then
sed -i -e s@/usr/local/mariadb/columnstore@$installdir@g /etc/my.cnf.d/columnstore.cnf
fi

127
oam/install_scripts/post-mysqld-install
View File

@ -38,132 +38,7 @@ for arg in "$@"; do
fi
done
#cd $installdir/mysql/lib
#for file in libmysqlclient libmysqlclient_r; do
# ln -sf $file.so.18.0.0 $file.so
# ln -sf $file.so.18.0.0 $file.so.18
#done
cd $installdir/mysql/lib/plugin
for file in ha_archive ha_blackhole ha_federated ha_innodb; do
ln -sf $file.so $file.so.0
done
### taken from MySQL-server-5.1.30-0.glibc23.i386.rpm
mysql_datadir=$installdir/mysql/db
# Create data directory if needed
if test ! -d $mysql_datadir; then mkdir -m 755 $mysql_datadir; fi
if test ! -d $mysql_datadir/mysql; then mkdir $mysql_datadir/mysql; fi
if test ! -d $mysql_datadir/test; then mkdir $mysql_datadir/test; fi
if test ! -d $installdir/mysql/lib; then mkdir $installdir/mysql/lib; fi
if test ! -d $installdir/mysql/lib/mysql; then mkdir $installdir/mysql/lib/mysql; fi
# Create a MySQL user and group. Do not report any problems if it already
# exists.
groupadd -r $user 2> /dev/null || true
useradd -m -r -d $mysql_datadir -s /bin/bash -c "MySQL server" -g $user $user 2> /dev/null || true
# The user may already exist, make sure it has the proper group nevertheless (BUG#12823)
usermod -g $user $user 2> /dev/null || true
# Change permissions so that the user that will run the MySQL daemon
# owns all database files.
chown -R $user:$user $installdir/mysql
# Initiate databases if needed
if [ $installdir != "/usr/local/mariadb/columnstore" ]; then
sed -i -e s@/usr/local/mariadb/columnstore@$installdir@g /etc/my.cnf.d/columnstore.cnf
fi
# InfiniDB testing hook...
test -x /usr/local/bin/idb-testing-mysql-pre-start && /usr/local/bin/idb-testing-mysql-pre-start $installdir/mysql
# If DB exist, run upgrade script if it exist
if [ -d $installdir/mysql/db/calpontsys ]; then
if [ -x $installdir/mysql/mysql-Columnstore ]; then
# Restart in the same way that mysqld will be started normally.
$installdir/mysql/mysql-Columnstore stop >/dev/null 2>&1
sleep 2
$installdir/mysql/mysql-Columnstore start --skip-grant-tables
sleep 5
# Run MariaDB (mysql) upgrade script, if it exist
if [ -x $installdir/mysql/bin/mysql_upgrade ]; then
echo "Running mysql_upgrade script"
if [[ ${password} == " " ]]; then
$installdir/mysql/bin/mysql_upgrade > $tmpdir/mysql_upgrade.log
if [ $? -ne 0 ]; then
echo "ERROR: mysql_upgrade failure, check $tmpdir/mysql_upgrade.log"
$installdir/mysql/mysql-Columnstore stop
sleep 2
exit 2;
fi
else
$installdir/mysql/bin/mysql_upgrade --password=$password > $tmpdir/mysql_upgrade.log
if [ $? -ne 0 ]; then
echo "ERROR: mysql_upgrade failure, check $tmpdir/mysql_upgrade.log"
$installdir/mysql/mysql-Columnstore stop
sleep 2
exit 2;
fi
fi
fi
# stop
$installdir/mysql/mysql-Columnstore stop
sleep 2
fi
fi
### Don't give the user the notes, we'll fix them ourselves...
$installdir/mysql/scripts/mysql_install_db --rpm --user=$user --basedir=$installdir/mysql >/dev/null
# Change permissions again to fix any new files.
chown -R $user:$user $mysql_datadir
# Fix permissions for the permission database so that only the user
# can read them.
chmod -R og-rw $mysql_datadir/mysql
# Change permissions again to fix any new files.
chown -R $user.$user $installdir/mysql
if [ -f $installdir/lib/libcalmysql.so.1.0.0 ]; then
libcalmysql=$installdir/lib/libcalmysql.so.1.0.0
libudfsdk=$installdir/lib/libudf_mysql.so.1.0.0
libregrsdk=$installdir/lib/libregr_mysql.so.1.0.0
is_columnstore_tables=$installdir/lib/is_columnstore_tables.so.1.0.0
is_columnstore_columns=$installdir/lib/is_columnstore_columns.so.1.0.0
is_columnstore_extents=$installdir/lib/is_columnstore_extents.so.1.0.0
is_columnstore_files=$installdir/lib/is_columnstore_files.so.1.0.0
elif [ -f $installdir/lib/libcalmysql.so.1 ]; then
libcalmysql=$installdir/lib/libcalmysql.so.1
libudfsdk=$installdir/lib/libudf_mysql.so.1
libregrsdk=$installdir/lib/libregr_mysql.so.1
is_columnstore_tables=$installdir/lib/is_columnstore_tables.so.1
is_columnstore_columns=$installdir/lib/is_columnstore_columns.so.1
is_columnstore_extents=$installdir/lib/is_columnstore_extents.so.1
is_columnstore_files=$installdir/lib/is_columnstore_files.so.1
else
libcalmysql=
fi
if [ -n "$libcalmysql" ]; then
cd $installdir/mysql/lib/plugin
ln -sf $libcalmysql libcalmysql.so
ln -sf $libcalmysql libcalmysqlent.so
ln -sf $libudfsdk libudf_mysql.so
ln -sf $libregrsdk libregr_mysql.so
ln -sf $is_columnstore_tables is_columnstore_tables.so
ln -sf $is_columnstore_columns is_columnstore_columns.so
ln -sf $is_columnstore_extents is_columnstore_extents.so
ln -sf $is_columnstore_files is_columnstore_files.so
fi
# cleanup previous install mysql replication files
rm -rf $mysql_datadir/*relay*
rm -rf $mysql_datadir/mysql-bin.*
rm -rf $mysql_datadir/master.info
mysql_install_db --rpm --user=$user >/dev/null
exit 0

6
oam/install_scripts/pre-uninstall.in
View File

@ -38,7 +38,7 @@ $installdir/myql/columnstore-Mysql stop > /dev/null 2>&1
if [ $user != "root" ]; then
export COLUMNSTORE_INSTALL_DIR=$installdir
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$installdir/lib:$installdir/mysql/lib
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$installdir/lib
fi
cloud=`$installdir/bin/getConfig Installation Cloud`
@ -72,8 +72,6 @@ if [ $user == "root" ]; then
rm -f /var/log/mariadb/columnstore/*.log1 > /dev/null 2>&1
fi
rm -rf $installdir/mysql/db/columnstore_log_archive > /dev/null 2>&1\
# delete Mariab Columnstore shared memory segments
$installdir/bin/clearShm > /dev/null 2>&1
@ -131,7 +129,7 @@ fi
if [ $quiet != 1 ]; then
#make copy of Columnstore.xml
/bin/cp -f @ENGINE_SYSCONFDIR@/columnstore/Columnstore.xml @ENGINE_SYSCONFDIR@/columnstore/Columnstore.xml.rpmsave > /dev/null 2>&1
/bin/cp -f $installdir/mysql/my.cnf $installdir/mysql/my.cnf.rpmsave > /dev/null 2>&1
/bin/cp -f /etc/my.cnf.d/columnstore.cnf /etc/my.cnf.d/columnstore.cnf.rpmsave > /dev/null 2>&1
cp $installdir/bin/myCnf-include-args.text $installdir/bin/myCnf-include-args.text.rpmsave >& /dev/null
rm -f @ENGINE_SYSCONFDIR@/columnstore/AlarmConfig.xml.installSave
fi

3
oam/install_scripts/rsync.sh
View File

@ -25,7 +25,8 @@ if { $PASSWORD == "ssh" } {
set PASSWORD ""
}
set COMMAND "rsync -vopgr -e ssh --exclude=mysql/ --exclude=test/ --exclude=infinidb_vtable/ --exclude=infinidb_querystats/ --exclude=calpontsys/ --include=*/ --include=*/* --exclude=* $INSTALLDIR/mysql/db/ $USERNAME@$SERVER:$INSTALLDIR/mysql/db/"
# LinuxJedi: Not sure what this is expecting to be copying, but it probably shouldn't be doing it
# set COMMAND "rsync -vopgr -e ssh --exclude=mysql/ --exclude=test/ --exclude=infinidb_vtable/ --exclude=infinidb_querystats/ --exclude=calpontsys/ --include=*/ --include=*/* --exclude=* $INSTALLDIR/mysql/db/ $USERNAME@$SERVER:$INSTALLDIR/mysql/db/"
#
# run command

8
oam/install_scripts/slave-rep-columnstore.sh
View File

@ -56,7 +56,7 @@ stop slave;
EOD
cat ${tmpdir}/idb_slave-rep.sql >>${tmpdir}/slave-rep-status.log
$installdir/mysql/bin/mysql \
mysql \
--user=root \
calpontsys <${tmpdir}/idb_slave-rep.sql >>${tmpdir}/slave-rep-status.log 2>&1
@ -78,7 +78,7 @@ CHANGE MASTER TO
EOD
cat ${tmpdir}/idb_slave-rep.sql >>${tmpdir}/slave-rep-status.log
$installdir/mysql/bin/mysql \
mysql \
--user=root \
calpontsys <${tmpdir}/idb_slave-rep.sql >>${tmpdir}/slave-rep-status.log 2>&1
@ -93,7 +93,7 @@ start slave;
EOD
cat ${tmpdir}/idb_slave-rep.sql >>${tmpdir}/slave-rep-status.log
$installdir/mysql/bin/mysql \
mysql \
--user=root \
calpontsys <${tmpdir}/idb_slave-rep.sql >>${tmpdir}/slave-rep-status.log 2>&1
@ -108,7 +108,7 @@ SHOW SLAVE STATUS\G
EOD
cat ${tmpdir}/idb_slave-rep.sql >>${tmpdir}/slave-rep-status.log
$installdir/mysql/bin/mysql \
mysql \
--user=root \
calpontsys <${tmpdir}/idb_slave-rep.sql >>${tmpdir}/slave-rep-status.log 2>&1