MCOL-4597: logic from postConfigure migrated to post-install to upgrade existing columnstore.cnf

debian/mariadb-plugin-columnstore.install

@@ -35,6 +35,7 @@ usr/bin/mcs-loadbrm.py
 usr/bin/mcs-stop-controllernode.sh
 usr/bin/mcsGetConfig
 usr/bin/mcsSetConfig
+usr/bin/mycnfUpgrade
 usr/bin/post-mysql-install
 usr/bin/post-mysqld-install
 usr/bin/reset_locks

oam/install_scripts/columnstore-post-install.in

@@ -92,6 +92,11 @@ if [ -f @ENGINE_SYSCONFDIR@/columnstore/storagemanager.cnf.rpmsave ]; then
     /bin/cp -f @ENGINE_SYSCONFDIR@/columnstore/storagemanager.cnf.rpmsave @ENGINE_SYSCONFDIR@/columnstore/storagemanager.cnf
 fi
 
+if [ -f @ENGINE_SYSCONFDIR@/columnstore/Columnstore.xml.rpmsave ]; then
+    /bin/cp -f @ENGINE_SYSCONFDIR@/columnstore/Columnstore.xml @ENGINE_SYSCONFDIR@/columnstore/Columnstore.xml.new
+    /bin/cp -f @ENGINE_SYSCONFDIR@/columnstore/Columnstore.xml.rpmsave @ENGINE_SYSCONFDIR@/columnstore/Columnstore.xml
+fi
+
 touch /dev/shm/columnstore-test && rm /dev/shm/columnstore-test
 if [ $? -ne 0 ] ; then
     echo "User $user will need R/W access to /dev/shm."
@@ -202,6 +207,14 @@ if [ $user = "root" ]; then
     fi
 fi
 
+# upgrade the columnstore.cnf file
+if [ -f @MARIADB_MYCNFDIR@/columnstore.cnf.rpmsave ]; then
+    @ENGINE_BINDIR@/mycnfUpgrade > @ENGINE_LOGDIR@/install/mycnfUpgrade.log 2>&1
+    if [ $? -ne 0 ]; then
+        echo "Error: Problem upgrade columnstore.cnf, check @ENGINE_LOGDIR@/install/mycnfUpgrade.log"
+    fi
+fi
+
 if [ $user = "root" ]; then
     @ENGINE_BINDIR@/columnstoreSyslogSetup.sh install > $installLogDir/syslog_install.log 2>&1
 

oam/install_scripts/columnstoreAlias

@@ -1,6 +1,9 @@
+#!/usr/bin/env bash
 # MariaDB Columnstore Alias Commands
-#
-alias ma=mcsadmin
+
+mcspm1host=$(mcsGetConfig PMS1 IPaddr)
+mcsapikey=$(grep -r 'x-api-key' /etc/columnstore/cmapi_server.conf | cut -d= -f2 | xargs)
+
 alias core='cd /var/log/mariadb/columnstore/corefiles'
 alias tdebug='tail -f /var/log/mariadb/columnstore/debug.log'
 alias tinfo='tail -f /var/log/mariadb/columnstore/info.log'
@@ -8,4 +11,9 @@ alias terror='tail -f /var/log/mariadb/columnstore/err.log'
 alias twarning='tail -f /var/log/mariadb/columnstore/warning.log'
 alias tcrit='tail -f /var/log/mariadb/columnstore/crit.log'
 alias dbrm='cd /var/lib/columnstore/data1/systemFiles/dbrm'
-alias mcsmodule='cat /var/lib/columnstore/local/module'
+alias mcsModule='cat /var/lib/columnstore/local/module'
+alias mcsStatus="curl -s https://$mcspm1host:8640/cmapi/0.4.0/cluster/status --header 'Content-Type:application/json' --header 'x-api-key:$mcsapikey' -k | jq ."
+alias mcsStart="curl -s -X PUT https://$mcspm1host:8640/cmapi/0.4.0/cluster/start --header 'Content-Type:application/json' --header 'x-api-key:$mcsapikey' --data '{\"timeout\":60}' -k | jq ."
+alias mcsShutdown="curl -s -X PUT https://$mcspm1host:8640/cmapi/0.4.0/cluster/shutdown --header 'Content-Type:application/json' --header 'x-api-key:$mcsapikey' --data '{\"timeout\":60}' -k | jq ."
+alias mcsReadOnly="curl -s -X PUT https://$mcspm1host:8640/cmapi/0.4.0/cluster/mode-set --header 'Content-Type:application/json' --header 'x-api-key:$mcsapikey' --data '{\"timeout\":20, \"mode\": \"readonly\"}' -k | jq ."
+alias mcsReadWrite="curl -s -X PUT https://$mcspm1host:8640/cmapi/0.4.0/cluster/mode-set --header 'Content-Type:application/json' --header 'x-api-key:$mcsapikey' --data '{\"timeout\":20, \"mode\": \"readwrite\"}' -k | jq ."

oam/install_scripts/myCnf-include-args.text

@@ -1,18 +1,32 @@
 # List of my.cnf arguments that should be checked and saved during upgrade install
 #
-columnstore_local_query
 log-bin
+log_bin
 server-id
+server_id
 max_length_for_sort_data
 tmpdir
 log-error
+log_error
 general_log_file
 slow_query_log_file
 general-log
+general_log
 slow-query-log
+slow_query_log
 character-set-server
+character_set_server
 collation-server
+collation_server
 init-connect
+init_connect
 binlog_format
 secure-auth
+secure_auth
 port
+log_bin_index 
+relay_log
+relay_log_index
+gtid_strict_mode
+log_slave_updates
+columnstore_*

oamapps/CMakeLists.txt

@@ -1,2 +1,3 @@
 
 add_subdirectory(columnstoreDB)
+add_subdirectory(postConfigure)

oamapps/postConfigure/CMakeLists.txt

@@ -0,0 +1,16 @@
+
+include_directories( ${ENGINE_COMMON_INCLUDES} )
+
+
+########### next target ###############
+
+set(mycnfUpgrade_SRCS mycnfUpgrade.cpp)
+
+add_executable(mycnfUpgrade ${mycnfUpgrade_SRCS})
+
+target_compile_options(mycnfUpgrade PRIVATE -Wno-unused-result)
+
+target_link_libraries(mycnfUpgrade ${ENGINE_LDFLAGS} ${ENGINE_READLINE_LIBRARY} ncurses ${MARIADB_CLIENT_LIBS} ${ENGINE_EXEC_LIBS})
+
+install(TARGETS mycnfUpgrade DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine)
+       

oamapps/postConfigure/mycnfUpgrade.cpp

@@ -0,0 +1,287 @@
+/* Copyright (C) 2014 InfiniDB, Inc.
+   Copyright (C) 2016 MariaDB Corporation
+
+   This program is free software; you can redistribute it and/or
+   modify it under the terms of the GNU General Public License
+   as published by the Free Software Foundation; version 2 of
+   the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+   MA 02110-1301, USA. */
+
+/******************************************************************************************
+* $Id: mycnfUpgrade.cpp 64 2006-10-12 22:21:51Z dhill $
+*
+*
+*
+******************************************************************************************/
+/**
+ * @file
+ */
+
+
+#include <unistd.h>
+#include <iterator>
+#include <numeric>
+#include <deque>
+#include <iostream>
+#include <ostream>
+#include <fstream>
+#include <cstdlib>
+#include <string>
+#include <limits.h>
+#include <sstream>
+#include <exception>
+#include <stdexcept>
+#include <vector>
+#include <stdio.h>
+#include <ctype.h>
+#include <netdb.h>
+#include <sys/sysinfo.h>
+#include <climits>
+#include <cstring>
+#include <glob.h>
+#include <boost/regex.hpp>
+#include "liboamcpp.h"
+#include "installdir.h"
+#include "mcsconfig.h"
+
+using namespace std;
+using namespace oam;
+
+
+/* MCOL-1844.  On an upgrade, the user may have customized options in their old 
+ * myCnf-include-args.text file.  Merge it with the packaged version, and then process as we
+ * have before.
+ */
+string rtrim(const string &in) {
+	string::const_reverse_iterator rbegin = in.rbegin();
+	while (rbegin != in.rend() && isspace(*rbegin))
+		++rbegin;
+	return string(in.begin(), rbegin.base());
+}
+
+void mergeMycnfIncludeArgs() 
+{
+	string userArgsFilename = std::string(MCSSUPPORTDIR) + "/myCnf-include-args.text.rpmsave";
+	string packagedArgsFilename = std::string(MCSSUPPORTDIR) + "/myCnf-include-args.text";
+	ifstream userArgs(userArgsFilename.c_str());
+	fstream packagedArgs(packagedArgsFilename.c_str(), ios::in);
+
+	if (!userArgs || !packagedArgs)
+		return;
+
+	// de-dup the args and comments in both files
+	set<string> argMerger;
+	set<string> comments;
+	string line;
+	while (getline(packagedArgs, line)) {
+		line = rtrim(line);
+		if (line[0] == '#')
+			comments.insert(line);
+		else if (line.size() > 0)
+			argMerger.insert(line);
+	}
+    while (getline(userArgs, line)) {
+        line = rtrim(line);
+        if (line[0] == '#')
+            comments.insert(line);
+        else if (line.size() > 0)
+            argMerger.insert(line);
+    }
+	userArgs.close();	
+	packagedArgs.close();
+
+	// write the merged version, comments first.  They'll get ordered 
+	// alphabetically but, meh.
+	packagedArgs.open(packagedArgsFilename.c_str(), ios::out | ios::trunc);
+	for (set<string>::iterator it = comments.begin(); it != comments.end(); it++)
+		packagedArgs << *it << endl;
+	for (set<string>::iterator it = argMerger.begin(); it != argMerger.end(); it++)
+		packagedArgs << *it << endl;
+	packagedArgs.close();
+}
+
+int main(int argc, char* argv[])
+{
+    Oam oam;
+
+    //check for port argument
+    string mysqlPort;
+
+    if (argc > 1)
+    {
+        mysqlPort = argv[1];
+
+        // set mysql password
+        oam.changeMyCnf( "port", mysqlPort );
+        exit (0);
+    }
+
+    //my.cnf file
+    string mycnfFile = std::string(MCSMYCNFDIR) + "/columnstore.cnf";
+    ifstream mycnffile (mycnfFile.c_str());
+
+    if (!mycnffile)
+    {
+        cerr << "mycnfUpgrade - columnstore.cnf file not found: " << mycnfFile << endl;
+        exit (1);
+    }
+
+    //my.cnf.rpmsave file
+    string mycnfsaveFile = std::string(MCSMYCNFDIR) + "/columnstore.cnf.rpmsave";
+    ifstream mycnfsavefile (mycnfsaveFile.c_str());
+
+    if (!mycnfsavefile)
+    {
+        cerr << "mycnfUpgrade - columnstore.cnf.rpmsave file not found: " << mycnfsaveFile << endl;
+        exit (1);
+    }
+
+	// MCOL-1844.  The user may have added options to their myCnf-include-args file.  Merge 
+	// myCnf-include-args.text with myCnf-include-args.text.rpmsave, save in myCnf-include-args.text
+	mergeMycnfIncludeArgs();
+
+    //include arguments file
+    string includeFile = std::string(MCSSUPPORTDIR) + "/myCnf-include-args.text";
+    ifstream includefile (includeFile.c_str());
+
+    if (!includefile)
+    {
+        cerr << "mycnfUpgrade - columnstore.cnf include argument file not found: " << includeFile << endl;
+        exit (1);
+    }
+
+    //exclude arguments file
+    string excludeFile = std::string(MCSSUPPORTDIR) + "/myCnf-exclude-args.text";
+    ifstream excludefile (excludeFile.c_str());
+
+    if (!excludefile)
+    {
+        cerr << "mycnfUpgrade - columnstore.cnf exclude argument file not found: " << endl;
+        exit (1);
+    }
+
+    //go though include list
+    char line[200];
+    string includeArg;
+
+    while (includefile.getline(line, 200))
+    {
+        includeArg = line;
+
+        boost::regex icludeArgRegEx("^#*\\s*" + includeArg + "\\s*=");
+        //see if in columnstore.cnf.rpmsave
+        ifstream mycnfsavefile (mycnfsaveFile.c_str());
+        char line[200];
+        string oldbuf;
+
+        while (mycnfsavefile.getline(line, 200))
+        {
+            oldbuf = line;
+
+            if ( boost::regex_search(oldbuf.begin(), oldbuf.end(), icludeArgRegEx) )
+            {
+                //found in columnstore.cnf.rpmsave, check if this is commented out
+                if ( line[0] != '#' )
+                {
+                    // no, check in columnstore.cnf and replace if exist or add if it doesn't
+
+                    ifstream mycnffile (mycnfFile.c_str());
+                    vector <string> lines;
+                    char line1[200];
+                    string newbuf;
+                    bool updated = false;
+
+                    while (mycnffile.getline(line1, 200))
+                    {
+                        newbuf = line1;
+
+                        if ( boost::regex_search(newbuf.begin(), newbuf.end(), icludeArgRegEx) )
+                        {
+                            newbuf = oldbuf;
+                            cout << "Updated argument: " << includeArg << endl;
+                            updated = true;
+                        }
+
+                        //output to temp file
+                        lines.push_back(newbuf);
+                    }
+
+                    //write out a new columnstore.cnf
+                    mycnffile.close();
+                    unlink (mycnfFile.c_str());
+                    ofstream newFile (mycnfFile.c_str());
+
+                    //create new file
+					int fd = open(mycnfFile.c_str(), O_RDWR|O_CREAT, 0644);
+
+                    copy(lines.begin(), lines.end(), ostream_iterator<string>(newFile, "\n"));
+                    newFile.close();
+
+                    close(fd);
+
+                    if (!updated)
+                    {
+                        //not found, so add
+                        ifstream mycnffile (mycnfFile.c_str());
+                        vector <string> lines;
+                        char line1[200];
+                        string newbuf;
+
+                        while (mycnffile.getline(line1, 200))
+                        {
+                            newbuf = line1;
+                            boost::regex mysqldSectionRegEx("\\[mysqld\\]");
+
+                            if ( boost::regex_search(newbuf.begin(), newbuf.end(), mysqldSectionRegEx) )
+                            {
+                                lines.push_back(newbuf);
+                                newbuf = oldbuf;
+                                cout << "Added argument: " << includeArg << endl;
+                            }
+
+                            //output to temp file
+                            lines.push_back(newbuf);
+                        }
+
+                        //write out a new columnstore.cnf
+                        mycnffile.close();
+                        unlink (mycnfFile.c_str());
+                        ofstream newFile (mycnfFile.c_str());
+
+                        //create new file
+                        int fd = open(mycnfFile.c_str(), O_RDWR | O_CREAT, 0666);
+
+                        copy(lines.begin(), lines.end(), ostream_iterator<string>(newFile, "\n"));
+                        newFile.close();
+
+                        close(fd);
+                        break;
+                    }
+                }
+            }
+        }
+    }
+
+	string USER = "mysql";
+	
+    char* p = getenv("USER");
+
+    if (p && *p)
+        USER = p;
+
+    string cmd = "chown " + USER + ":" + USER + " " + mycnfFile;
+    system(cmd.c_str());
+
+    exit (0);
+}
+// vim:ts=4 sw=4:
+