From f966584bee4a7e7b33d24492f49e1d326744b6a2 Mon Sep 17 00:00:00 2001 From: Jose Rojas Date: Wed, 23 Sep 2020 10:19:19 -0500 Subject: [PATCH 1/6] MCOL-4314 shared storage manager cluster initialization fix. --- .../columnstore-post-install.in | 27 ++++++++++--------- .../mariadb-columnstore-start.sh | 1 + oam/install_scripts/mcs-loadbrm.py.in | 18 ++++++++++--- 3 files changed, 30 insertions(+), 16 deletions(-) diff --git a/oam/install_scripts/columnstore-post-install.in b/oam/install_scripts/columnstore-post-install.in index 2f033c357..9ffb904a6 100755 --- a/oam/install_scripts/columnstore-post-install.in +++ b/oam/install_scripts/columnstore-post-install.in @@ -302,25 +302,26 @@ if [ ! -z "$MCS_USE_S3_STORAGE" ] && [ $MCS_USE_S3_STORAGE -eq 1 ]; then fi #change ownership/permissions to be able to run columnstore as non-root -chown -R mysql:mysql /var/log/mariadb/columnstore -chown -R mysql:mysql /etc/columnstore -chown -R mysql:mysql /var/lib/columnstore -chown -R mysql:mysql /tmp/columnstore_tmp_files -chmod 777 /tmp/columnstore_tmp_files -chmod 777 /dev/shm +# TODO: Remove conditional once container dispatcher uses non-root by default +if [ $(running_systemd) -eq 0 ]; then + chown -R mysql:mysql /var/log/mariadb/columnstore + chown -R mysql:mysql /etc/columnstore + chown -R mysql:mysql /var/lib/columnstore + chown -R mysql:mysql /tmp/columnstore_tmp_files + chmod 777 /tmp/columnstore_tmp_files + chmod 777 /dev/shm +fi systemctl cat mariadb-columnstore.service > /dev/null 2>&1 if [ $? -eq 0 ] && [ $(running_systemd) -eq 0 ]; then - # prevent clusters using shared storage from initializing columnstore more than once - IFLAG=/var/lib/columnstore/storagemanager/cs-initialized mkdir -p /var/lib/columnstore/storagemanager chown -R mysql:mysql /var/lib/columnstore/storagemanager - if [ ! -e $IFLAG ]; then - touch $IFLAG - echo "Populating the engine initial system catalog." - systemctl start mariadb-columnstore - fi + echo "Populating the engine initial system catalog." + + # prevent nodes using shared storage manager from stepping on each other when initializing + flock /var/lib/columnstore/storagemanager -c "systemctl start mariadb-columnstore" + systemctl stop mariadb-columnstore # shutdown after initialization fi if [ $stop_mysqld -eq 1 ];then diff --git a/oam/install_scripts/mariadb-columnstore-start.sh b/oam/install_scripts/mariadb-columnstore-start.sh index 79d094ba3..34325fdfd 100644 --- a/oam/install_scripts/mariadb-columnstore-start.sh +++ b/oam/install_scripts/mariadb-columnstore-start.sh @@ -9,6 +9,7 @@ /bin/systemctl start mcs-exemgr /bin/systemctl start mcs-dmlproc /bin/systemctl start mcs-ddlproc +sleep 2 su -s /bin/sh -c 'dbbuilder 7' mysql 2> /tmp/columnstore_tmp_files/dbbuilder.log diff --git a/oam/install_scripts/mcs-loadbrm.py.in b/oam/install_scripts/mcs-loadbrm.py.in index 53d0d5614..dc64946cf 100755 --- a/oam/install_scripts/mcs-loadbrm.py.in +++ b/oam/install_scripts/mcs-loadbrm.py.in @@ -81,7 +81,12 @@ if __name__ == '__main__': # Single-node on S3 if storage.lower() == 's3' and not bucket.lower() == 'some_bucket' and pmCount == 1: try: - brm_saves_current = subprocess.check_output(['smcat', brm]) + if use_systemd: + args = ['su', '-s', '/bin/sh', '-c', f'smcat {brm}', 'mysql'] + else: + args = ['smcat', brm] + + brm_saves_current = subprocess.check_output(args) except subprocess.CalledProcessError as e: # will happen when brm file does not exist print('{} does not exist.'.format(brm), file=sys.stderr) @@ -122,6 +127,8 @@ node.".format(e)) if not os.path.exists(dbrmroot): os.makedirs(dbrmroot) + if use_systemd: + shutil.chown(dbrmroot, USER, GROUP) current_name = '{}_{}'.format(dbrmroot, e) @@ -145,7 +152,11 @@ node {}.'.format(primary_address), file=sys.stderr) print('{} does not exist.'.format(brm), file=sys.stderr) if brm_saves_current: - cmd = '{} {}{}'.format(loadbrm, dbrmroot, \ + if use_systemd: + cmd = 'su -s /bin/sh -c "{} {}{}" mysql'.format(loadbrm, dbrmroot, \ +brm_saves_current.decode("utf-8").replace("BRM_saves", "")) + else: + cmd = '{} {}{}'.format(loadbrm, dbrmroot, \ brm_saves_current.decode("utf-8").replace("BRM_saves", "")) try: retcode = subprocess.call(cmd, shell=True) @@ -153,6 +164,7 @@ brm_saves_current.decode("utf-8").replace("BRM_saves", "")) print('{} exits with {}.'.format(cmd, retcode)) sys.exit(1) for shm_file in glob.glob('/dev/shm/*'): - shutil.chown(shm_file, USER, GROUP) + if use_systemd: + shutil.chown(shm_file, USER, GROUP) except OSError as e: sys.exit(1) From a957352aafe47d9b7386003a25d0053a174f9f1a Mon Sep 17 00:00:00 2001 From: Jose Rojas Date: Wed, 23 Sep 2020 17:09:52 -0500 Subject: [PATCH 2/6] MCOL-4012 Related. Minor improvement. --- oam/install_scripts/mcs-loadbrm.py.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/oam/install_scripts/mcs-loadbrm.py.in b/oam/install_scripts/mcs-loadbrm.py.in index dc64946cf..fcde3064f 100755 --- a/oam/install_scripts/mcs-loadbrm.py.in +++ b/oam/install_scripts/mcs-loadbrm.py.in @@ -163,8 +163,8 @@ brm_saves_current.decode("utf-8").replace("BRM_saves", "")) if retcode < 0: print('{} exits with {}.'.format(cmd, retcode)) sys.exit(1) - for shm_file in glob.glob('/dev/shm/*'): - if use_systemd: + if use_systemd: + for shm_file in glob.glob('/dev/shm/*'): shutil.chown(shm_file, USER, GROUP) except OSError as e: sys.exit(1) From bb7f137f65f1ab6c7fe43235f754f7713045755f Mon Sep 17 00:00:00 2001 From: Jose Rojas Date: Thu, 24 Sep 2020 10:22:05 -0500 Subject: [PATCH 3/6] MCOL-4314 Move flock to mariadb-columnstore-start script --- oam/install_scripts/columnstore-post-install.in | 6 ++---- oam/install_scripts/mariadb-columnstore-start.sh | 7 ++++++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/oam/install_scripts/columnstore-post-install.in b/oam/install_scripts/columnstore-post-install.in index 9ffb904a6..5eacf56ac 100755 --- a/oam/install_scripts/columnstore-post-install.in +++ b/oam/install_scripts/columnstore-post-install.in @@ -318,10 +318,8 @@ if [ $? -eq 0 ] && [ $(running_systemd) -eq 0 ]; then chown -R mysql:mysql /var/lib/columnstore/storagemanager echo "Populating the engine initial system catalog." - - # prevent nodes using shared storage manager from stepping on each other when initializing - flock /var/lib/columnstore/storagemanager -c "systemctl start mariadb-columnstore" - systemctl stop mariadb-columnstore # shutdown after initialization + systemctl start mariadb-columnstore + systemctl stop mariadb-columnstore fi if [ $stop_mysqld -eq 1 ];then diff --git a/oam/install_scripts/mariadb-columnstore-start.sh b/oam/install_scripts/mariadb-columnstore-start.sh index 34325fdfd..94f2cb1d5 100644 --- a/oam/install_scripts/mariadb-columnstore-start.sh +++ b/oam/install_scripts/mariadb-columnstore-start.sh @@ -2,6 +2,10 @@ # This script allows to gracefully start MCS +# prevent nodes using shared storage manager from stepping on each other when initializing +# flock -x 200 will open up an exclusive file lock to run atomic operations +( +flock -x 200 /bin/systemctl start mcs-workernode /bin/systemctl start mcs-controllernode /bin/systemctl start mcs-primproc @@ -11,6 +15,7 @@ /bin/systemctl start mcs-ddlproc sleep 2 -su -s /bin/sh -c 'dbbuilder 7' mysql 2> /tmp/columnstore_tmp_files/dbbuilder.log +su -s /bin/sh -c 'dbbuilder 7' mysql 1> /tmp/columnstore_tmp_files/dbbuilder.log +) 200>/var/lib/columnstore/storagemanager/cs-initialized exit 0 From 040f9fd32cc3491e9f27b8949199e210681ab899 Mon Sep 17 00:00:00 2001 From: Jose Rojas Date: Thu, 24 Sep 2020 15:14:09 -0500 Subject: [PATCH 4/6] Do not invoke subshell when using flock --- oam/install_scripts/mariadb-columnstore-start.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/oam/install_scripts/mariadb-columnstore-start.sh b/oam/install_scripts/mariadb-columnstore-start.sh index 94f2cb1d5..733ed4fdf 100644 --- a/oam/install_scripts/mariadb-columnstore-start.sh +++ b/oam/install_scripts/mariadb-columnstore-start.sh @@ -3,9 +3,10 @@ # This script allows to gracefully start MCS # prevent nodes using shared storage manager from stepping on each other when initializing -# flock -x 200 will open up an exclusive file lock to run atomic operations -( -flock -x 200 +# flock will open up an exclusive file lock to run atomic operations +exec {fd_lock}>/var/lib/columnstore/storagemanager/storagemanager-lock +flock -x "$fd_lock" + /bin/systemctl start mcs-workernode /bin/systemctl start mcs-controllernode /bin/systemctl start mcs-primproc @@ -14,8 +15,8 @@ flock -x 200 /bin/systemctl start mcs-dmlproc /bin/systemctl start mcs-ddlproc sleep 2 - su -s /bin/sh -c 'dbbuilder 7' mysql 1> /tmp/columnstore_tmp_files/dbbuilder.log -) 200>/var/lib/columnstore/storagemanager/cs-initialized + +flock -u "$fd_lock" exit 0 From 5fc6885af28d4c64cc73a61a1cde9e5584201a6a Mon Sep 17 00:00:00 2001 From: Jose Rojas Date: Thu, 24 Sep 2020 19:49:37 -0500 Subject: [PATCH 5/6] MCOL-4314 shared storage manager prevent data corruption on init --- oam/install_scripts/mariadb-columnstore-start.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oam/install_scripts/mariadb-columnstore-start.sh b/oam/install_scripts/mariadb-columnstore-start.sh index 733ed4fdf..aaeaacd7e 100644 --- a/oam/install_scripts/mariadb-columnstore-start.sh +++ b/oam/install_scripts/mariadb-columnstore-start.sh @@ -5,7 +5,7 @@ # prevent nodes using shared storage manager from stepping on each other when initializing # flock will open up an exclusive file lock to run atomic operations exec {fd_lock}>/var/lib/columnstore/storagemanager/storagemanager-lock -flock -x "$fd_lock" +flock -n "$fd_lock" || exit 0 /bin/systemctl start mcs-workernode /bin/systemctl start mcs-controllernode From f129d506d3e18c6133bc487992c40f2a630e541f Mon Sep 17 00:00:00 2001 From: Jose Rojas Date: Fri, 25 Sep 2020 09:48:58 -0500 Subject: [PATCH 6/6] MCOL-4314 Address comment regarding systemctl stop mariadb-columnstore --- oam/install_scripts/columnstore-post-install.in | 1 - 1 file changed, 1 deletion(-) diff --git a/oam/install_scripts/columnstore-post-install.in b/oam/install_scripts/columnstore-post-install.in index 5eacf56ac..930c6a929 100755 --- a/oam/install_scripts/columnstore-post-install.in +++ b/oam/install_scripts/columnstore-post-install.in @@ -319,7 +319,6 @@ if [ $? -eq 0 ] && [ $(running_systemd) -eq 0 ]; then echo "Populating the engine initial system catalog." systemctl start mariadb-columnstore - systemctl stop mariadb-columnstore fi if [ $stop_mysqld -eq 1 ];then