MCOL-307 merge with develop

.gitignore

@@ -98,6 +98,7 @@ utils/loggingcpp/messageids.h
 CPackConfig.cmake
 CPackSourceConfig.cmake
 build/columnstore.community.spec
+build/releasenum
 *.rpm
 install_manifest_libs.txt
 install_manifest_platform.txt

CMakeLists.txt

@@ -36,6 +36,31 @@ ENDIF(NOT CMAKE_BUILD_TYPE)
 
 #set( CMAKE_VERBOSE_MAKEFILE on )
 
+IF (NOT VERSION)
+    SET (VERSION 1.0.5)
+ENDIF()
+IF (NOT RELEASE)
+    SET (RELEASE 1)
+ENDIF()
+
+SET (PACKAGE columnstore)
+SET (PACKAGE_NAME columnstore)
+SET (PACKAGE_TARNAME columnstore)
+SET (PACKAGE_BUGREPORT support@mariadb.com)
+SET (PACKAGE_URL "")
+
+IF (NOT PACKAGE_VERSION)
+    SET (PACKAGE_VERSION ${VERSION})
+ENDIF()
+IF (NOT PACKAGE_RELEASE)
+    SET (PACKAGE_RELEASE ${RELEASE})
+ENDIF()
+SET (PACKAGE_STRING columnstore-${PACKAGE_VERSION})
+
+
+SET (ENGINE_SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR})
+SET (INSTALL_ENGINE "/usr/local/mariadb/columnstore")
+
 # use, i.e. don't skip the full RPATH for the build tree
 SET(CMAKE_SKIP_BUILD_RPATH  FALSE)
 
@@ -43,31 +68,18 @@ SET(CMAKE_SKIP_BUILD_RPATH  FALSE)
 # (but later on when installing)
 SET(CMAKE_BUILD_WITH_INSTALL_RPATH FALSE)
 
-SET(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib")
+SET(CMAKE_INSTALL_RPATH "${INSTALL_ENGINE}/lib")
 
 # add the automatically determined parts of the RPATH
 # which point to directories outside the build tree to the install RPATH
 SET(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
 
-
 # the RPATH to be used when installing, but only if it's not a system directory
-LIST(FIND CMAKE_PLATFORM_IMPLICIT_LINK_DIRECTORIES "${CMAKE_INSTALL_PREFIX}/lib" isSystemDir)
+LIST(FIND CMAKE_PLATFORM_IMPLICIT_LINK_DIRECTORIES "${INSTALL_ENGINE}/lib" isSystemDir)
 IF("${isSystemDir}" STREQUAL "-1")
-   SET(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib")
+    SET(CMAKE_INSTALL_RPATH "${INSTALL_ENGINE}/lib")
 ENDIF("${isSystemDir}" STREQUAL "-1")
 
-SET (PACKAGE columnstore)
-SET (VERSION 1.0.2)
-SET (PACKAGE_NAME columnstore)
-SET (PACKAGE_TARNAME columnstore)
-SET (PACKAGE_VERSION 1.0.2)
-SET (PACKAGE_STRING columnstore-1.0.2)
-SET (PACKAGE_BUGREPORT support@mariadb.com)
-SET (PACKAGE_URL "")
-
-SET (ENGINE_SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR})
-SET (INSTALL_ENGINE "/usr/local/mariadb/columnstore")
-
 INCLUDE (configureEngine.cmake)
 
 CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/build/releasenum.in ${CMAKE_CURRENT_BINARY_DIR}/build/releasenum @ONLY IMMEDIATE)
@@ -110,7 +122,7 @@ ENDFOREACH()
 SET(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -ggdb3 -fno-tree-vectorize -DSAFE_MUTEX -DSAFEMALLOC -DENABLED_DEBUG_SYNC -O0 -Wall -D_DEBUG -DHAVE_CONFIG_H")
 SET(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -ggdb3 -fno-tree-vectorize -DSAFE_MUTEX -DSAFEMALLOC -DENABLED_DEBUG_SYNC -O0 -Wall -D _DEBUG -DHAVE_CONFIG_H")
 
-SET (ENGINE_LDFLAGS    "-Wl,--rpath -Wl,${INSTALL_ENGINE}/lib -Wl,--no-as-needed -Wl,--add-needed")  
+SET (ENGINE_LDFLAGS    "-Wl,--no-as-needed -Wl,--add-needed")  
 
 FIND_PACKAGE(Boost 1.53.0 REQUIRED COMPONENTS system filesystem thread regex date_time) 
 

README

@@ -1,6 +1,6 @@
 This is MariaDB ColumnStore 1.0.4
 MariaDB ColumnStore 1.0.4 is the development version of MariaDB ColumnStore. 
-It is built by porting InfiniDB 4.6.2 on MariaDB 10.1.18 and adding entirely 
+It is built by porting InfiniDB 4.6.2 on MariaDB 10.1.19 and adding entirely 
 new features not found anywhere else.
 
 MariaDB ColumnStore 1.0.4 is an Beta release. This is the first MariaDB 

README.md

@@ -1,9 +1,9 @@
-#MariaDB ColumnStore Storage/Execution engine 1.0.4
+#MariaDB ColumnStore Storage/Execution engine 1.0.6
-MariaDB ColumnStore 1.0.4 is the development version of MariaDB ColumnStore. 
+MariaDB ColumnStore 1.0.6 is the development version of MariaDB ColumnStore. 
-It is built by porting InfiniDB 4.6.2 on MariaDB 10.1.18 and adding entirely 
+It is built by porting InfiniDB 4.6.2 on MariaDB 10.1.19 and adding entirely 
 new features not found anywhere else.
 
-#MariaDB ColumnStore 1.0.4 is an Beta release. 
+#MariaDB ColumnStore 1.0.6 is an GA release. 
 This is the first MariaDB ColumnStore release, not all features planned for the MariaDB ColumnStore 1.0 
 series are included in this release. 
 

build/releasenum

@@ -1,2 +0,0 @@
-version=1.0.4
-release=1

build/releasenum.in

@@ -1,2 +1,2 @@
-version=@CPACK_RPM_PACKAGE_VERSION@
+version=@VERSION@
-release=@CPACK_RPM_PACKAGE_RELEASE@
+release=@RELEASE@

cpackEngineRPM.cmake

@@ -15,10 +15,10 @@ SET(CPACK_PACKAGE_NAME "mariadb-columnstore")
 SET(ENGINE_ARCH "x86_64")
 
 IF (NOT CPACK_RPM_PACKAGE_VERSION)
-SET (CPACK_RPM_PACKAGE_VERSION "1.0.0")
+SET (CPACK_RPM_PACKAGE_VERSION ${PACKAGE_VERSION})
 ENDIF()
 IF (NOT CPACK_RPM_PACKAGE_RELEASE)
-SET (CPACK_RPM_PACKAGE_RELEASE "0")
+SET (CPACK_RPM_PACKAGE_RELEASE ${PACKAGE_RELEASE})
 ENDIF()
 
 SET(CPACK_RPM_PACKAGE_NAME ${CPACK_PACKAGE_NAME})
@@ -194,8 +194,8 @@ SET(CPACK_RPM_platform_USER_FILELIST
 "/usr/local/mariadb/columnstore/bin/autoConfigure"
 "/usr/local/mariadb/columnstore/bin/ddlcleanup"
 "/usr/local/mariadb/columnstore/bin/idbmeminfo"
-"/usr/local/mariadb/columnstore/bin/IDBInstanceCmds.sh"
+"/usr/local/mariadb/columnstore/bin/MCSInstanceCmds.sh"
-"/usr/local/mariadb/columnstore/bin/IDBVolumeCmds.sh"
+"/usr/local/mariadb/columnstore/bin/MCSVolumeCmds.sh"
 "/usr/local/mariadb/columnstore/bin/binary_installer.sh" 
 "/usr/local/mariadb/columnstore/bin/myCnf-include-args.text" 
 "/usr/local/mariadb/columnstore/bin/myCnf-exclude-args.text"
@@ -318,6 +318,18 @@ SET(CPACK_RPM_storage-engine_USER_FILELIST
 "/usr/local/mariadb/columnstore/lib/libudf_mysql.so.1.0.0"
 "/usr/local/mariadb/columnstore/lib/libudf_mysql.so.1"
 "/usr/local/mariadb/columnstore/lib/libudf_mysql.so"
+"/usr/local/mariadb/columnstore/lib/is_columnstore_columns.so"
+"/usr/local/mariadb/columnstore/lib/is_columnstore_columns.so.1"
+"/usr/local/mariadb/columnstore/lib/is_columnstore_columns.so.1.0.0"
+"/usr/local/mariadb/columnstore/lib/is_columnstore_extents.so"
+"/usr/local/mariadb/columnstore/lib/is_columnstore_extents.so.1"
+"/usr/local/mariadb/columnstore/lib/is_columnstore_extents.so.1.0.0"
+"/usr/local/mariadb/columnstore/lib/is_columnstore_tables.so"
+"/usr/local/mariadb/columnstore/lib/is_columnstore_tables.so.1"
+"/usr/local/mariadb/columnstore/lib/is_columnstore_tables.so.1.0.0"
+"/usr/local/mariadb/columnstore/lib/is_columnstore_files.so"
+"/usr/local/mariadb/columnstore/lib/is_columnstore_files.so.1"
+"/usr/local/mariadb/columnstore/lib/is_columnstore_files.so.1.0.0"
 "/usr/local/mariadb/columnstore/mysql/mysql-Columnstore"
 "/usr/local/mariadb/columnstore/mysql/install_calpont_mysql.sh"
 "/usr/local/mariadb/columnstore/mysql/syscatalog_mysql.sql"
@@ -326,6 +338,7 @@ SET(CPACK_RPM_storage-engine_USER_FILELIST
 "/usr/local/mariadb/columnstore/mysql/calsetuserpriority.sql"
 "/usr/local/mariadb/columnstore/mysql/calremoveuserpriority.sql"
 "/usr/local/mariadb/columnstore/mysql/calshowprocesslist.sql"
+"/usr/local/mariadb/columnstore/mysql/columnstore_info.sql"
 ${ignored})
 
 

dbcon/execplan/calpontsystemcatalog.cpp

@@ -2599,6 +2599,7 @@ const vector< pair<CalpontSystemCatalog::OID, CalpontSystemCatalog::TableName> >
     SimpleColumn *c1 = new SimpleColumn(CALPONT_SCHEMA+"."+SYSTABLE_TABLE+"."+TABLENAME_COL, fSessionID);
     SimpleColumn *c2 = new SimpleColumn(CALPONT_SCHEMA+"."+SYSTABLE_TABLE+"."+SCHEMA_COL, fSessionID);
     SimpleColumn *c3 = new SimpleColumn(CALPONT_SCHEMA+"."+SYSTABLE_TABLE+"."+OBJECTID_COL, fSessionID);
+    SimpleColumn *c4 = new SimpleColumn(CALPONT_SCHEMA+"."+SYSTABLE_TABLE+"."+CREATEDATE_COL, fSessionID);
     
     SRCP srcp;
     srcp.reset(c1);
@@ -2607,6 +2608,8 @@ const vector< pair<CalpontSystemCatalog::OID, CalpontSystemCatalog::TableName> >
     colMap.insert(CMVT_(CALPONT_SCHEMA+"."+SYSTABLE_TABLE+"."+SCHEMA_COL, srcp));
     srcp.reset(c3);
     colMap.insert(CMVT_(CALPONT_SCHEMA+"."+SYSTABLE_TABLE+"."+OBJECTID_COL, srcp));
+    srcp.reset(c4);
+    colMap.insert(CMVT_(CALPONT_SCHEMA+"."+SYSTABLE_TABLE+"."+CREATEDATE_COL, srcp));
     csep.columnMapNonStatic(colMap);
         
     srcp.reset(c1->clone());
@@ -2653,6 +2656,14 @@ const vector< pair<CalpontSystemCatalog::OID, CalpontSystemCatalog::TableName> >
           tables[i].second.schema = (*it)->GetStringData(i);
       }
     }
+    for (it = sysDataList.begin(); it != sysDataList.end(); it++)
+    {
+        if ((*it)->ColumnOID() == c4->oid())
+        {
+            for (int i = 0; i < (*it)->dataCount(); i++)
+                tables[i].second.create_date = (*it)->GetData(i);
+        }
+    }
     
     for (it = sysDataList.begin(); it != sysDataList.end(); it++)
     {

dbcon/execplan/calpontsystemcatalog.h

@@ -431,6 +431,7 @@ public:
 		TableName(const TableAliasName& tan): schema(tan.schema), table(tan.table) {}
 		std::string schema;
 		std::string table;
+        int64_t create_date;
 		bool operator<(const TableName& rhs) const;
 		bool operator>=(const TableName& rhs) const { return !(*this < rhs); }
 		bool operator==(const TableName& rhs) const

dbcon/mysql/CMakeLists.txt

@@ -17,7 +17,7 @@ SET ( libcalmysql_SRCS
     ha_view.cpp sm.cpp
     ha_window_function.cpp
     ha_calpont_partition.cpp
-    ha_pseudocolumn.cpp )
+    ha_pseudocolumn.cpp)
 
 add_definitions(-DMYSQL_DYNAMIC_PLUGIN)
 
@@ -29,12 +29,58 @@ target_link_libraries(calmysql ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${NETSNMP_
 
 set_target_properties(calmysql PROPERTIES VERSION 1.0.0 SOVERSION 1)
 
-install(TARGETS calmysql DESTINATION ${ENGINE_LIBDIR} COMPONENT storage-engine)
+SET ( is_columnstore_tables_SRCS
+    is_columnstore_tables.cpp
+    )
+add_library(is_columnstore_tables SHARED ${is_columnstore_tables_SRCS})
+
+target_link_libraries(is_columnstore_tables ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} threadpool)
+
+# Don't prepend .so file with 'lib'
+set_target_properties(is_columnstore_tables PROPERTIES PREFIX "")
+set_target_properties(is_columnstore_tables PROPERTIES VERSION 1.0.0 SOVERSION 1)
+
+SET ( is_columnstore_columns_SRCS
+    is_columnstore_columns.cpp
+    )
+add_library(is_columnstore_columns SHARED ${is_columnstore_columns_SRCS})
+
+target_link_libraries(is_columnstore_columns ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} threadpool)
+
+# Don't prepend .so file with 'lib'
+set_target_properties(is_columnstore_columns PROPERTIES PREFIX "")
+set_target_properties(is_columnstore_columns PROPERTIES VERSION 1.0.0 SOVERSION 1)
+
+SET ( is_columnstore_extents_SRCS
+    is_columnstore_extents.cpp
+    )
+add_library(is_columnstore_extents SHARED ${is_columnstore_extents_SRCS})
+
+target_link_libraries(is_columnstore_extents ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} threadpool)
+
+# Don't prepend .so file with 'lib'
+set_target_properties(is_columnstore_extents PROPERTIES PREFIX "")
+set_target_properties(is_columnstore_extents PROPERTIES VERSION 1.0.0 SOVERSION 1)
+
+SET ( is_columnstore_files_SRCS
+    is_columnstore_files.cpp
+    )
+add_library(is_columnstore_files SHARED ${is_columnstore_files_SRCS})
+
+target_link_libraries(is_columnstore_files ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} threadpool)
+
+# Don't prepend .so file with 'lib'
+set_target_properties(is_columnstore_files PROPERTIES PREFIX "")
+set_target_properties(is_columnstore_files PROPERTIES VERSION 1.0.0 SOVERSION 1)
+
+
+install(TARGETS calmysql is_columnstore_tables is_columnstore_columns is_columnstore_extents is_columnstore_files DESTINATION ${ENGINE_LIBDIR} COMPONENT storage-engine)
 install(FILES syscatalog_mysql.sql 
               dumpcat_mysql.sql 
               calsetuserpriority.sql 
               calremoveuserpriority.sql 
               calshowprocesslist.sql 
+              columnstore_info.sql
               my.cnf
               DESTINATION ${ENGINE_MYSQLDIR} COMPONENT storage-engine)
 install(PROGRAMS install_calpont_mysql.sh mysql-Columnstore dumpcat.pl

dbcon/mysql/columnstore_info.sql

@@ -0,0 +1,50 @@
+CREATE DATABASE columnstore_info;
+USE columnstore_info;
+
+DELIMITER //
+CREATE FUNCTION `format_filesize`(filesize FLOAT) RETURNS varchar(20) CHARSET utf8 DETERMINISTIC
+BEGIN
+
+DECLARE n INT DEFAULT 1;
+
+LOOP
+    IF filesize < 1024 THEN
+        RETURN concat(round(filesize, 2), ' ', elt(n, 'Bytes', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB', 'BB'));
+    END IF;
+    SET filesize = filesize / 1024;
+    SET n = n + 1;
+END LOOP;
+
+END //
+
+
+CREATE PROCEDURE total_usage ()
+BEGIN
+    SELECT
+        (SELECT format_filesize(sum(data_size)) TOTAL_DATA_SIZE FROM INFORMATION_SCHEMA.COLUMNSTORE_EXTENTS) TOTAL_DATA_SIZE,
+        (SELECT format_filesize(sum(file_size)) TOTAL_DISK_USAGE FROM INFORMATION_SCHEMA.COLUMNSTORE_FILES) TOTAL_DISK_USAGE;
+END //
+
+CREATE PROCEDURE table_usage (IN t_name char(64))
+BEGIN
+    IF t_name IS NOT NULL THEN
+    SELECT TABLE_NAME, format_filesize(sum(cf.file_size)) DATA_DISK_USAGE, format_filesize(sum(IFNULL(ccf.file_size, 0))) DICT_DISK_USAGE, format_filesize(sum(cf.file_size) + sum(IFNULL(ccf.file_size, 0))) TOTAL_USAGE FROM INFORMATION_SCHEMA.COLUMNSTORE_COLUMNS cc
+JOIN INFORMATION_SCHEMA.COLUMNSTORE_FILES cf ON cc.object_id = cf.object_id
+LEFT JOIN INFORMATION_SCHEMA.COLUMNSTORE_FILES ccf ON cc.dictionary_object_id = ccf.object_id
+WHERE table_name = t_name GROUP BY table_name;
+    ELSE
+    SELECT TABLE_NAME, format_filesize(sum(cf.file_size)) DATA_DISK_USAGE, format_filesize(sum(IFNULL(ccf.file_size, 0))) DICT_DISK_USAGE, format_filesize(sum(cf.file_size) + sum(IFNULL(ccf.file_size, 0))) TOTAL_USAGE FROM INFORMATION_SCHEMA.COLUMNSTORE_COLUMNS cc
+JOIN INFORMATION_SCHEMA.COLUMNSTORE_FILES cf ON cc.object_id = cf.object_id
+LEFT JOIN INFORMATION_SCHEMA.COLUMNSTORE_FILES ccf ON cc.dictionary_object_id = ccf.object_id
+GROUP BY table_name;
+    END IF;
+END //
+    
+CREATE PROCEDURE compression_ratio()
+BEGIN
+SELECT CONCAT(((sum(compressed_data_size) / sum(data_size)) * 100), '%') COMPRESSION_RATIO FROM INFORMATION_SCHEMA.COLUMNSTORE_EXTENTS ce
+JOIN INFORMATION_SCHEMA.COLUMNSTORE_FILES cf ON ce.object_id = cf.object_id
+WHERE compressed_data_size IS NOT NULL;
+END //
+
+DELIMITER ;

dbcon/mysql/install_calpont_mysql.sh

@@ -30,6 +30,10 @@ df=$installdir/mysql/my.cnf
 $installdir/mysql/bin/mysql --defaults-file=$df --force --user=root $pwprompt mysql 2>/tmp/mysql_install.log <<EOD
 INSTALL PLUGIN columnstore SONAME 'libcalmysql.so';
 INSTALL PLUGIN infinidb SONAME 'libcalmysql.so';
+INSTALL PLUGIN columnstore_tables SONAME 'is_columnstore_tables.so';
+INSTALL PLUGIN columnstore_columns SONAME 'is_columnstore_columns.so';
+INSTALL PLUGIN columnstore_extents SONAME 'is_columnstore_extents.so';
+INSTALL PLUGIN columnstore_files SONAME 'is_columnstore_files.so';
 -- these are deprecated names
 DELETE FROM mysql.func WHERE name='caldisablepartition';
 DELETE FROM mysql.func WHERE name='caldroppartition';
@@ -130,6 +134,7 @@ $installdir/mysql/bin/mysql --defaults-file=$df --user=root $pwprompt mysql 2>/d
 $installdir/mysql/bin/mysql --defaults-file=$df --user=root $pwprompt mysql 2>/dev/null <$installdir/mysql/calsetuserpriority.sql
 $installdir/mysql/bin/mysql --defaults-file=$df --user=root $pwprompt mysql 2>/dev/null <$installdir/mysql/calremoveuserpriority.sql
 $installdir/mysql/bin/mysql --defaults-file=$df --user=root $pwprompt mysql 2>/dev/null <$installdir/mysql/calshowprocesslist.sql
+$installdir/mysql/bin/mysql --defaults-file=$df --user=root $pwprompt mysql 2>/dev/null <$installdir/mysql/columnstore_info.sql
 
 sed -i 's/infinidb_compression_type=1/infinidb_compression_type=2/' $installdir/mysql/my.cnf >/dev/null 2>&1
 

dbcon/mysql/is_columnstore_columns.cpp

@@ -0,0 +1,187 @@
+/* c-basic-offset: 4; tab-width: 4; indent-tabs-mode: nil
+ * vi: set shiftwidth=4 tabstop=4 expandtab:
+ *  :indentSize=4:tabSize=4:noTabs=true:
+ *
+ * Copyright (C) 2016 MariaDB Corporaton
+ *
+ * This program is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; version 2 of
+ * the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#include "idb_mysql.h"
+#include <vector>
+#include <limits>
+
+#include <boost/shared_ptr.hpp>
+#include "calpontsystemcatalog.h"
+#include "dataconvert.h"
+
+
+// Required declaration as it isn't in a MairaDB include
+bool schema_table_store_record(THD *thd, TABLE *table);
+
+ST_FIELD_INFO is_columnstore_columns_fields[] =
+{
+    {"TABLE_SCHEMA", 64, MYSQL_TYPE_STRING, 0, 0, 0, 0},
+    {"TABLE_NAME", 64, MYSQL_TYPE_STRING, 0, 0, 0, 0},
+    {"COLUMN_NAME", 64, MYSQL_TYPE_STRING, 0, 0, 0, 0},
+    {"OBJECT_ID", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0},
+    {"DICTIONARY_OBJECT_ID", 11, MYSQL_TYPE_LONG, 0, MY_I_S_MAYBE_NULL, 0, 0},
+    {"LIST_OBJECT_ID", 11, MYSQL_TYPE_LONG, 0, MY_I_S_MAYBE_NULL, 0, 0},
+    {"TREE_OBJECT_ID", 11, MYSQL_TYPE_LONG, 0, MY_I_S_MAYBE_NULL, 0, 0},
+    {"DATA_TYPE", 64, MYSQL_TYPE_STRING, 0, 0, 0, 0},
+    {"COLUMN_LENGTH", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0},
+    {"COLUMN_POSITION", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0},
+    {"COLUMN_DEFAULT", 0, MYSQL_TYPE_LONG_BLOB, 0, MY_I_S_MAYBE_NULL, 0, 0},
+    {"IS_NULLABLE", 1, MYSQL_TYPE_TINY, 0, 0, 0, 0},
+    {"NUMERIC_PRECISION", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0},
+    {"NUMERIC_SCALE", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0},
+    {"IS_AUTOINCREMENT", 1, MYSQL_TYPE_TINY, 0, 0, 0, 0},
+    {"COMPRESSION_TYPE", 64, MYSQL_TYPE_STRING, 0, 0, 0, 0},
+    {0, 0, MYSQL_TYPE_NULL, 0, 0, 0, 0}
+
+};
+
+static int is_columnstore_columns_fill(THD *thd, TABLE_LIST *tables, COND *cond)
+{
+    CHARSET_INFO *cs = system_charset_info;
+    TABLE *table = tables->table;
+
+    boost::shared_ptr<execplan::CalpontSystemCatalog> systemCatalogPtr =
+        execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(execplan::CalpontSystemCatalog::idb_tid2sid(thd->thread_id));
+
+    const std::vector< std::pair<execplan::CalpontSystemCatalog::OID, execplan::CalpontSystemCatalog::TableName> > catalog_tables
+        = systemCatalogPtr->getTables();
+
+    for (std::vector<std::pair<execplan::CalpontSystemCatalog::OID, execplan::CalpontSystemCatalog::TableName> >::const_iterator it = catalog_tables.begin();
+         it != catalog_tables.end(); ++it)
+    {
+        execplan::CalpontSystemCatalog::RIDList column_rid_list = systemCatalogPtr->columnRIDs((*it).second);
+        for (size_t col_num = 0; col_num < column_rid_list.size(); col_num++)
+        {
+            execplan::CalpontSystemCatalog::TableColName tcn = systemCatalogPtr->colName(column_rid_list[col_num].objnum);
+            execplan::CalpontSystemCatalog::ColType ct = systemCatalogPtr->colType(column_rid_list[col_num].objnum);
+
+			table->field[0]->store(tcn.schema.c_str(), tcn.schema.length(), cs);
+			table->field[1]->store(tcn.table.c_str(), tcn.table.length(), cs);
+			table->field[2]->store(tcn.column.c_str(), tcn.column.length(), cs);
+			table->field[3]->store(column_rid_list[col_num].objnum);
+			if (ct.ddn.dictOID == std::numeric_limits<int32_t>::min())
+			{
+			    table->field[4]->set_null();
+			}
+			else
+			{
+			    table->field[4]->set_notnull();
+			    table->field[4]->store(ct.ddn.dictOID);
+			}
+			if (ct.ddn.listOID == std::numeric_limits<int32_t>::min())
+			{
+			    table->field[5]->set_null();
+			}
+			else
+			{
+			    table->field[5]->set_notnull();
+			    table->field[5]->store(ct.ddn.listOID);
+			}
+			if (ct.ddn.treeOID == std::numeric_limits<int32_t>::min())
+			{
+			    table->field[6]->set_null();
+			}
+			else
+			{
+			    table->field[6]->set_notnull();
+			    table->field[6]->store(ct.ddn.treeOID);
+			}
+			std::string data_type = execplan::colDataTypeToString(ct.colDataType);
+			table->field[7]->store(data_type.c_str(), data_type.length(), cs);
+			table->field[8]->store(ct.colWidth);
+			table->field[9]->store(ct.colPosition);
+			if (ct.defaultValue.empty())
+			{
+			    table->field[10]->set_null();
+			}
+			else
+			{
+			    table->field[10]->set_notnull();
+    			table->field[10]->store(ct.defaultValue.c_str(), ct.defaultValue.length(), cs);
+    		}
+    		table->field[11]->store(ct.autoincrement);
+			table->field[12]->store(ct.precision);
+			table->field[13]->store(ct.scale);
+			if (ct.constraintType != execplan::CalpontSystemCatalog::NOTNULL_CONSTRAINT)
+			{
+			    table->field[14]->store(true);
+			}
+			else
+			{
+			    table->field[14]->store(false);
+			}
+
+			std::string compression_type;
+			switch (ct.compressionType)
+			{
+				case 0:
+				    compression_type = "None";
+				    break;
+				case 2:
+				    compression_type = "Snappy";
+				    break;
+				default:
+				    compression_type = "Unknown";
+				    break;
+			}
+			table->field[15]->store(compression_type.c_str(), compression_type.length(), cs);
+            if (schema_table_store_record(thd, table))
+                return 1;
+        }
+    }
+
+
+
+    return 0;
+}
+
+static int is_columnstore_columns_plugin_init(void *p)
+{
+    ST_SCHEMA_TABLE *schema = (ST_SCHEMA_TABLE*) p;
+    schema->fields_info = is_columnstore_columns_fields;
+    schema->fill_table = is_columnstore_columns_fill;
+    return 0;
+}
+
+static struct st_mysql_information_schema is_columnstore_columns_plugin_version =
+{ MYSQL_INFORMATION_SCHEMA_INTERFACE_VERSION };
+
+maria_declare_plugin(is_columnstore_columns_plugin)
+{
+    MYSQL_INFORMATION_SCHEMA_PLUGIN,
+    &is_columnstore_columns_plugin_version,
+    "COLUMNSTORE_COLUMNS",
+    "MariaDB Corporaton",
+    "An information schema plugin to list ColumnStore columns",
+    PLUGIN_LICENSE_GPL,
+    is_columnstore_columns_plugin_init,
+    //is_columnstore_tables_plugin_deinit,
+    NULL,
+    0x0100,
+    NULL,
+    NULL,
+    "1.0",
+    MariaDB_PLUGIN_MATURITY_STABLE
+}
+maria_declare_plugin_end;
+
+

dbcon/mysql/is_columnstore_extents.cpp

@@ -0,0 +1,202 @@
+/* c-basic-offset: 4; tab-width: 4; indent-tabs-mode: nil
+ * vi: set shiftwidth=4 tabstop=4 expandtab:
+ *  :indentSize=4:tabSize=4:noTabs=true:
+ *
+ * Copyright (C) 2016 MariaDB Corporaton
+ *
+ * This program is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; version 2 of
+ * the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#include "idb_mysql.h"
+#include <vector>
+#include <limits>
+
+#include "dbrm.h"
+#include "objectidmanager.h"
+
+
+// Required declaration as it isn't in a MairaDB include
+bool schema_table_store_record(THD *thd, TABLE *table);
+
+ST_FIELD_INFO is_columnstore_extents_fields[] =
+{
+    {"OBJECT_ID", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0},                     // 0
+    {"OBJECT_TYPE", 64, MYSQL_TYPE_STRING, 0, 0, 0, 0},                 // 1
+    {"LOGICAL_BLOCK_START", 19, MYSQL_TYPE_LONGLONG, 0, 0, 0, 0},       // 2
+    {"LOGICAL_BLOCK_END", 19, MYSQL_TYPE_LONGLONG, 0, 0, 0, 0},         // 3
+    {"MIN_VALUE", 19, MYSQL_TYPE_LONGLONG, 0, MY_I_S_MAYBE_NULL, 0, 0}, // 4
+    {"MAX_VALUE", 19, MYSQL_TYPE_LONGLONG, 0, MY_I_S_MAYBE_NULL, 0, 0}, // 5
+    {"WIDTH", 1, MYSQL_TYPE_TINY, 0, 0, 0, 0},                          // 6
+    {"DBROOT", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0},                        // 7
+    {"PARTITION_ID", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0},                  // 8
+    {"SEGMENT_ID", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0},                    // 9
+    {"BLOCK_OFFSET", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0},                  // 10
+    {"MAX_BLOCKS", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0},                    // 11
+    {"HIGH_WATER_MARK", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0},               // 12
+    {"STATE", 64, MYSQL_TYPE_STRING, 0, 0, 0, 0},                       // 13
+    {"STATUS", 64, MYSQL_TYPE_STRING, 0, 0, 0, 0},                      // 14
+    {"DATA_SIZE", 19, MYSQL_TYPE_LONGLONG, 0, 0, 0, 0},                 // 15
+    {0, 0, MYSQL_TYPE_NULL, 0, 0, 0, 0}
+};
+
+static int is_columnstore_extents_fill(THD *thd, TABLE_LIST *tables, COND *cond)
+{
+    CHARSET_INFO *cs = system_charset_info;
+    TABLE *table = tables->table;
+    std::vector<struct BRM::EMEntry> entries;
+   	std::vector<struct BRM::EMEntry>::iterator iter;
+	std::vector<struct BRM::EMEntry>::iterator end;
+
+    BRM::DBRM *emp = new BRM::DBRM();
+    if (!emp || !emp->isDBRMReady())
+    {
+        return 1;
+    }
+
+    execplan::ObjectIDManager oidm;
+    BRM::OID_t MaxOID = oidm.size();
+
+    for(BRM::OID_t oid = 3000; oid <= MaxOID; oid++)
+    {
+        emp->getExtents(oid, entries, false, false, true);
+        if (entries.size() == 0)
+            continue;
+
+        iter = entries.begin();
+        end = entries.end();
+        while (iter != end)
+        {
+            table->field[0]->store(oid);
+            if (iter->colWid > 0)
+            {
+                table->field[1]->store("Column", strlen("Column"), cs);
+                if (iter->partition.cprange.lo_val == std::numeric_limits<int64_t>::max() ||
+        			iter->partition.cprange.lo_val <= (std::numeric_limits<int64_t>::min() + 2))
+        		{
+        		    table->field[4]->set_null();
+        		}
+        		else
+        		{
+        		    table->field[4]->set_notnull();
+                    table->field[4]->store(iter->partition.cprange.lo_val);
+                }
+                if (iter->partition.cprange.hi_val == std::numeric_limits<int64_t>::max() ||
+        			iter->partition.cprange.hi_val <= (std::numeric_limits<int64_t>::min() + 2))
+        		{
+        		    table->field[5]->set_null();
+        		}
+        		else
+        		{
+                    table->field[5]->set_notnull();
+                    table->field[5]->store(iter->partition.cprange.hi_val);
+                }
+                table->field[6]->store(iter->colWid);
+
+            }
+            else
+            {
+                table->field[1]->store("Dictionary", strlen("Dictionary"), cs);
+                table->field[4]->set_null();
+                table->field[5]->set_null();
+                table->field[6]->store(8192);
+            }
+            table->field[2]->store(iter->range.start);
+            table->field[3]->store(iter->range.start + (iter->range.size * 1024) - 1);
+
+            table->field[7]->store(iter->dbRoot);
+            table->field[8]->store(iter->partitionNum);
+            table->field[9]->store(iter->segmentNum);
+            table->field[10]->store(iter->blockOffset);
+            table->field[11]->store(iter->range.size * 1024);
+            table->field[12]->store(iter->HWM);
+
+            switch (iter->partition.cprange.isValid)
+            {
+                case 0:
+                    table->field[13]->store("Invalid", strlen("Invalid"), cs);
+                    break;
+                case 1:
+                    table->field[13]->store("Updating", strlen("Updating"), cs);
+                    break;
+                case 2:
+                    table->field[13]->store("Valid", strlen("Valid"), cs);
+                    break;
+                default:
+                    table->field[13]->store("Unknown", strlen("Unknown"), cs);
+                    break;
+            }
+            switch (iter->status)
+            {
+                case BRM::EXTENTAVAILABLE:
+                    table->field[14]->store("Available", strlen("Available"), cs);
+                    break;
+                case BRM::EXTENTUNAVAILABLE:
+                    table->field[14]->store("Unavailable", strlen("Unavailable"), cs);
+                    break;
+                case BRM::EXTENTOUTOFSERVICE:
+                    table->field[14]->store("Out of service", strlen("Out of service"), cs);
+                    break;
+                default:
+                    table->field[14]->store("Unknown", strlen("Unknown"), cs);
+            }
+            table->field[15]->store((iter->HWM - iter->blockOffset + 1) * 8192);
+
+            if (schema_table_store_record(thd, table))
+            {
+                delete emp;
+                return 1;
+            }
+
+            iter++;
+
+        }
+    }
+
+    delete emp;
+    return 0;
+}
+
+static int is_columnstore_extents_plugin_init(void *p)
+{
+    ST_SCHEMA_TABLE *schema = (ST_SCHEMA_TABLE*) p;
+    schema->fields_info = is_columnstore_extents_fields;
+    schema->fill_table = is_columnstore_extents_fill;
+    return 0;
+}
+
+static struct st_mysql_information_schema is_columnstore_extents_plugin_version =
+{ MYSQL_INFORMATION_SCHEMA_INTERFACE_VERSION };
+
+maria_declare_plugin(is_columnstore_extents_plugin)
+{
+    MYSQL_INFORMATION_SCHEMA_PLUGIN,
+    &is_columnstore_extents_plugin_version,
+    "COLUMNSTORE_EXTENTS",
+    "MariaDB Corporaton",
+    "An information schema plugin to list ColumnStore extents",
+    PLUGIN_LICENSE_GPL,
+    is_columnstore_extents_plugin_init,
+    //is_columnstore_extents_plugin_deinit,
+    NULL,
+    0x0100,
+    NULL,
+    NULL,
+    "1.0",
+    MariaDB_PLUGIN_MATURITY_STABLE
+}
+maria_declare_plugin_end;
+
+

dbcon/mysql/is_columnstore_files.cpp

@@ -0,0 +1,163 @@
+/* c-basic-offset: 4; tab-width: 4; indent-tabs-mode: nil
+ * vi: set shiftwidth=4 tabstop=4 expandtab:
+ *  :indentSize=4:tabSize=4:noTabs=true:
+ *
+ * Copyright (C) 2016 MariaDB Corporaton
+ *
+ * This program is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; version 2 of
+ * the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#include "idb_mysql.h"
+
+#include "dbrm.h"
+#include "objectidmanager.h"
+#include "we_convertor.h"
+#include "we_define.h"
+#include "IDBPolicy.h"
+#include "we_config.h"
+#include "we_brm.h"
+
+// Required declaration as it isn't in a MairaDB include
+bool schema_table_store_record(THD *thd, TABLE *table);
+
+ST_FIELD_INFO is_columnstore_files_fields[] =
+{
+    {"OBJECT_ID", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0},
+    {"SEGMENT_ID", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0},
+    {"PARTITION_ID", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0},
+    {"FILENAME", 1024, MYSQL_TYPE_STRING, 0, 0, 0, 0},
+    {"FILE_SIZE", 19, MYSQL_TYPE_LONGLONG, 0, MY_I_S_MAYBE_NULL, 0, 0},
+    {"COMPRESSED_DATA_SIZE", 19, MYSQL_TYPE_LONGLONG, 0, MY_I_S_MAYBE_NULL, 0, 0},
+    {0, 0, MYSQL_TYPE_NULL, 0, 0, 0, 0}
+};
+
+
+static bool is_columnstore_files_get_entries(THD *thd, TABLE_LIST *tables, BRM::OID_t oid, std::vector<struct BRM::EMEntry> &entries)
+{
+    CHARSET_INFO *cs = system_charset_info;
+    TABLE *table = tables->table;
+
+    char oidDirName[WriteEngine::FILE_NAME_SIZE];
+	char fullFileName[WriteEngine::FILE_NAME_SIZE];
+    char dbDir[WriteEngine::MAX_DB_DIR_LEVEL][WriteEngine::MAX_DB_DIR_NAME_SIZE];
+    WriteEngine::Config config;
+	config.initConfigCache();
+
+    std::vector<struct BRM::EMEntry>::const_iterator iter = entries.begin();
+    while ( iter != entries.end() ) //organize extents into files
+    {
+        // Don't include files more than once at different block offsets
+        if (iter->blockOffset > 0)
+        {
+            iter++;
+            continue;
+        }
+        table->field[0]->store(oid);
+        table->field[1]->store(iter->segmentNum);
+        table->field[2]->store(iter->partitionNum);
+
+        WriteEngine::Convertor::oid2FileName(oid, oidDirName, dbDir, iter->partitionNum, iter->segmentNum);
+        snprintf(fullFileName, WriteEngine::FILE_NAME_SIZE, "%s/%s", WriteEngine::Config::getDBRootByNum(iter->dbRoot).c_str(), oidDirName);
+        table->field[3]->store(fullFileName, strlen(fullFileName), cs);
+
+        if (idbdatafile::IDBPolicy::exists(fullFileName))
+        {
+            table->field[4]->set_notnull();
+            table->field[4]->store(idbdatafile::IDBPolicy::size(fullFileName));
+            off64_t comp_size;
+            if ((comp_size = idbdatafile::IDBPolicy::compressedSize(fullFileName)) != -1)
+            {
+                table->field[5]->set_notnull();
+                table->field[5]->store(comp_size);
+            }
+            else
+            {
+                table->field[5]->set_null();
+            }
+        }
+        else
+        {
+            table->field[4]->set_null();
+            table->field[5]->set_null();
+        }
+
+        if (schema_table_store_record(thd, table))
+            return 1;
+        iter++;
+    }
+    return 0;
+}
+
+static int is_columnstore_files_fill(THD *thd, TABLE_LIST *tables, COND *cond)
+{
+    BRM::DBRM *emp = new BRM::DBRM();
+    std::vector<struct BRM::EMEntry> entries;
+
+    if (!emp || !emp->isDBRMReady())
+    {
+        return 1;
+    }
+
+    execplan::ObjectIDManager oidm;
+    BRM::OID_t MaxOID = oidm.size();
+
+    for(BRM::OID_t oid = 3000; oid <= MaxOID; oid++)
+    {
+        emp->getExtents(oid, entries, false, false, true);
+        if (entries.size() == 0)
+            continue;
+
+        if (is_columnstore_files_get_entries(thd, tables, oid, entries))
+        {
+            delete emp;
+            return 1;
+        }
+
+    }
+
+    delete emp;
+    return 0;
+}
+
+static int is_columnstore_files_plugin_init(void *p)
+{
+    ST_SCHEMA_TABLE *schema = (ST_SCHEMA_TABLE*) p;
+    schema->fields_info = is_columnstore_files_fields;
+    schema->fill_table = is_columnstore_files_fill;
+    return 0;
+}
+
+static struct st_mysql_information_schema is_columnstore_files_plugin_version =
+{ MYSQL_INFORMATION_SCHEMA_INTERFACE_VERSION };
+
+maria_declare_plugin(is_columnstore_files_plugin)
+{
+    MYSQL_INFORMATION_SCHEMA_PLUGIN,
+    &is_columnstore_files_plugin_version,
+    "COLUMNSTORE_FILES",
+    "MariaDB Corporaton",
+    "An information schema plugin to list ColumnStore filess",
+    PLUGIN_LICENSE_GPL,
+    is_columnstore_files_plugin_init,
+    //is_columnstore_files_plugin_deinit,
+    NULL,
+    0x0100,
+    NULL,
+    NULL,
+    "1.0",
+    MariaDB_PLUGIN_MATURITY_STABLE
+}
+maria_declare_plugin_end;

dbcon/mysql/is_columnstore_tables.cpp

@@ -0,0 +1,113 @@
+/* c-basic-offset: 4; tab-width: 4; indent-tabs-mode: nil
+ * vi: set shiftwidth=4 tabstop=4 expandtab:
+ *  :indentSize=4:tabSize=4:noTabs=true:
+ *
+ * Copyright (C) 2016 MariaDB Corporaton
+ *
+ * This program is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; version 2 of
+ * the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#include "idb_mysql.h"
+#include <vector>
+
+#include <boost/shared_ptr.hpp>
+#include "calpontsystemcatalog.h"
+#include "dataconvert.h"
+
+
+// Required declaration as it isn't in a MairaDB include
+bool schema_table_store_record(THD *thd, TABLE *table);
+
+ST_FIELD_INFO is_columnstore_tables_fields[] =
+{
+    {"TABLE_SCHEMA", 64, MYSQL_TYPE_STRING, 0, 0, 0, 0},
+    {"TABLE_NAME", 64, MYSQL_TYPE_STRING, 0, 0, 0, 0},
+    {"OBJECT_ID", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0},
+    {"CREATION_DATE", 0, MYSQL_TYPE_DATE, 0, 0, 0, 0},
+    {"COLUMN_COUNT", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0},
+    {"AUTOINCREMENT", 11, MYSQL_TYPE_LONG, 0, MY_I_S_MAYBE_NULL, 0, 0},
+    {0, 0, MYSQL_TYPE_NULL, 0, 0, 0, 0}
+};
+
+static int is_columnstore_tables_fill(THD *thd, TABLE_LIST *tables, COND *cond)
+{
+    CHARSET_INFO *cs = system_charset_info;
+    TABLE *table = tables->table;
+
+    boost::shared_ptr<execplan::CalpontSystemCatalog> systemCatalogPtr =
+        execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(execplan::CalpontSystemCatalog::idb_tid2sid(thd->thread_id));
+
+    const std::vector< std::pair<execplan::CalpontSystemCatalog::OID, execplan::CalpontSystemCatalog::TableName> > catalog_tables
+        = systemCatalogPtr->getTables();
+
+    for (std::vector<std::pair<execplan::CalpontSystemCatalog::OID, execplan::CalpontSystemCatalog::TableName> >::const_iterator it = catalog_tables.begin();
+         it != catalog_tables.end(); ++it)
+    {
+        execplan::CalpontSystemCatalog::TableInfo tb_info = systemCatalogPtr->tableInfo((*it).second);
+        std::string create_date = dataconvert::DataConvert::dateToString((*it).second.create_date);
+        table->field[0]->store((*it).second.schema.c_str(), (*it).second.schema.length(), cs);
+        table->field[1]->store((*it).second.table.c_str(), (*it).second.table.length(), cs);
+        table->field[2]->store((*it).first);
+        table->field[3]->store(create_date.c_str(), create_date.length(), cs);
+        table->field[4]->store(tb_info.numOfCols);
+        if (tb_info.tablewithautoincr)
+        {
+            table->field[5]->set_notnull();
+            table->field[5]->store(systemCatalogPtr->nextAutoIncrValue((*it).second));
+        }
+        else
+        {
+            table->field[5]->set_null();
+        }
+        table->field[5]->store(tb_info.tablewithautoincr);
+        if (schema_table_store_record(thd, table))
+            return 1;
+    }
+
+    return 0;
+}
+
+static int is_columnstore_tables_plugin_init(void *p)
+{
+    ST_SCHEMA_TABLE *schema = (ST_SCHEMA_TABLE*) p;
+    schema->fields_info = is_columnstore_tables_fields;
+    schema->fill_table = is_columnstore_tables_fill;
+    return 0;
+}
+
+static struct st_mysql_information_schema is_columnstore_tables_plugin_version =
+{ MYSQL_INFORMATION_SCHEMA_INTERFACE_VERSION };
+
+maria_declare_plugin(is_columnstore_tables_plugin)
+{
+    MYSQL_INFORMATION_SCHEMA_PLUGIN,
+    &is_columnstore_tables_plugin_version,
+    "COLUMNSTORE_TABLES",
+    "MariaDB Corporaton",
+    "An information schema plugin to list ColumnStore tables",
+    PLUGIN_LICENSE_GPL,
+    is_columnstore_tables_plugin_init,
+    //is_columnstore_tables_plugin_deinit,
+    NULL,
+    0x0100,
+    NULL,
+    NULL,
+    "1.0",
+    MariaDB_PLUGIN_MATURITY_STABLE
+}
+maria_declare_plugin_end;
+
+

oam/cloud/CMakeLists.txt

@@ -1,2 +1,2 @@
 
-install(PROGRAMS  IDBInstanceCmds.sh IDBVolumeCmds.sh DESTINATION ${ENGINE_BINDIR} COMPONENT platform)
+install(PROGRAMS  MCSInstanceCmds.sh MCSVolumeCmds.sh DESTINATION ${ENGINE_BINDIR} COMPONENT platform)

oam/cloud/MCSInstanceCmds.sh

@@ -0,0 +1,574 @@
+#! /bin/sh
+# MCSInstanceCmds.sh
+# get-local-instance-ID, get-zone, getPrivateIP from a Cloud environment
+#
+# 1. Amazon EC2
+
+prefix=/usr/local
+
+#check command
+if [ "$1" = "" ]; then
+	echo "Enter Command Name: {launchInstance|getInstance|getZone|getPrivateIP|getKey|getAMI|getType|terminateInstance|startInstance|assignElasticIP|deassignElasticIP|getProfile|stopInstance|getGroup|getSubnet}
+}"
+	exit 1
+fi
+
+if [ "$1" = "getPrivateIP" ]; then
+	if [ "$2" = "" ]; then
+		echo "Enter Instance Name"
+		exit 1
+	fi
+	instanceName="$2"
+fi
+
+if [ "$1" = "launchInstance" ]; then
+	if [ "$2" = "" ]; then
+		IPaddress="unassigned"
+	else
+		IPaddress="$2"
+	fi
+	if [ "$3" = "" ]; then
+		instanceType="unassigned"
+	else
+		instanceType="$3"
+	fi
+	if [ "$4" = "" ]; then
+		group="unassigned"
+	else
+		group="$4"
+	fi
+fi
+
+if [ "$1" = "terminateInstance" ]; then
+	if [ "$2" = "" ]; then
+		echo "Enter Instance Name"
+		exit 1
+	fi
+	instanceName="$2"
+fi
+
+if [ "$1" = "stopInstance" ]; then
+	if [ "$2" = "" ]; then
+		echo "Enter Instance Name"
+		exit 1
+	fi
+	instanceName="$2"
+fi
+
+if [ "$1" = "startInstance" ]; then
+	if [ "$2" = "" ]; then
+		echo "Enter Instance Name"
+		exit 1
+	fi
+	instanceName="$2"
+fi
+
+if [ "$1" = "assignElasticIP" ]; then
+	if [ "$2" = "" ]; then
+		echo "Enter Instance Name"
+		exit 1
+	else
+		instanceName="$2"
+	fi
+	if [ "$3" = "" ]; then
+		echo "Enter Elastic IP Address"
+		exit 1
+	else
+		IPAddress="$3"
+	fi
+fi
+
+if [ "$1" = "deassignElasticIP" ]; then
+	if [ "$2" = "" ]; then
+		echo "Enter Elastic IP Address"
+		exit 1
+	else
+		IPAddress="$2"
+	fi
+fi
+
+
+test -f /usr/local/mariadb/columnstore/post/functions && . /usr/local/mariadb/columnstore/post/functions
+
+ec2=`$prefix/mariadb/columnstore/bin/getConfig Installation EC2_HOME`
+
+if [ $ec2 == "unassigned" ]; then
+	if [ "$1" = "getPrivateIP" ]; then
+		echo "stopped"
+		exit 1
+	else
+       echo "unknown"
+        exit 1
+	fi
+fi
+
+java=`$prefix/mariadb/columnstore/bin/getConfig Installation JAVA_HOME`
+path=`$prefix/mariadb/columnstore/bin/getConfig Installation EC2_PATH`
+
+export PATH=$path
+export EC2_HOME=$ec2
+export JAVA_HOME=$java
+
+# get Keys and region
+AmazonAccessKeyFile=`$prefix/mariadb/columnstore/bin/getConfig Installation AmazonAccessKey`
+if [ $AmazonAccessKeyFile == "unassigned" ]; then
+	echo "FAILED: missing Config Setting AmazonAccessKey : $AmazonAccessKeyfile"
+	exit 1
+fi
+
+AmazonSecretKeyFile=`$prefix/mariadb/columnstore/bin/getConfig Installation AmazonSecretKey`
+if [ $AmazonSecretKeyFile == "unassigned" ]; then
+	echo "FAILED: missing Config Setting AmazonSecretKeyFile : $AmazonSecretKeyFile"
+	exit 1
+fi
+
+AmazonAccessKey=`cat $AmazonAccessKeyFile`
+AmazonSecretKey=`cat $AmazonSecretKeyFile`
+
+Region=`$prefix/mariadb/columnstore/bin/getConfig Installation AmazonRegion`
+subnet=`$prefix/mariadb/columnstore/bin/getConfig Installation AmazonSubNetID`
+
+if test ! -f $AmazonAccessKeyfile ; then
+	echo "FAILED: missing AmazonAccessKeyfile : $AmazonAccessKeyfile"
+	exit 1
+fi
+
+if test ! -f $AmazonSecretKeyfile ; then
+	echo "FAILED: missing AmazonSecretKeyfile : $AmazonSecretKeyfile"
+	exit 1
+fi
+
+#default instance to null
+instance=""
+
+describeInstanceFile="/tmp/describeInstance.txt"
+touch $describeInstanceFile
+
+describeInstance() {
+	ec2-describe-instances -O $AmazonAccessKey -W $AmazonSecretKey --region $Region > $describeInstanceFile 2>&1
+}
+
+#call at start up
+describeInstance
+
+getInstance() {
+	if [ "$instance" != "" ]; then
+		echo $instance
+		return
+	fi
+	
+	# first get local IP Address
+	localIP=`ifconfig eth0 | grep -m 1 "inet" | awk '{print substr($2,0,20)}'`
+
+	#get local Instance ID
+	instance=`cat $describeInstanceFile | grep -m 1 -w $localIP |  awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'`
+	if [ "$instance" == "" ]; then
+		describeInstance
+	fi
+	instance=`cat $describeInstanceFile | grep -m 1 -w $localIP |  awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'`
+
+	echo $instance
+	return
+}
+
+getInstancePrivate() {
+	if [ "$instance" != "" ]; then
+		echo $instance
+		return
+	fi
+	
+	# first get local IP Address
+        localIP=`ifconfig eth0 | grep -m 1 "inet" | awk '{print substr($2,0,20)}'`
+
+	#get local Instance ID
+	instance=`cat $describeInstanceFile | grep -m 1 -w $localIP |  awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'`
+	if [ "$instance" == "" ]; then
+		describeInstance
+	fi
+	instance=`cat $describeInstanceFile | grep -m 1 -w $localIP |  awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'`
+
+	return
+}
+
+
+getZone() {
+	#get from Columnstore.xml if it's there, if not, get from instance then store
+	zone=`$prefix/mariadb/columnstore/bin/getConfig Installation AmazonZone`
+
+	if [ "$zone" = "unassigned" ] || [ "$zone" = "" ]; then
+		#get local Instance ID
+		getInstancePrivate >/dev/null 2>&1
+		#get zone
+		if [ "$subnet" == "unassigned" ]; then
+			zone=`cat $describeInstanceFile | grep -m 1 $instance |  awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $11}'`
+			if [ "$zone" == "" ]; then
+				describeInstance
+			fi
+			zone=`cat $describeInstanceFile | grep -m 1 $instance |  awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $11}'`
+
+		else
+			zone=`cat $describeInstanceFile | grep -m 1 $instance |  awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $11}'`
+			if [ "$zone" == "" ]; then
+				describeInstance
+			fi
+			zone=`cat $describeInstanceFile | grep -m 1 $instance |  awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $11}'`
+		fi
+		$prefix/mariadb/columnstore/bin/setConfig Installation AmazonZone $zone
+	fi
+
+	echo $zone
+	return
+}
+
+getPrivateIP() {
+	#get instance info
+	grep -B1 -A4 -m 1 $instanceName $describeInstanceFile > /tmp/instanceInfo_$instanceName 2>&1
+	if [ `cat /tmp/instanceInfo_$instanceName | wc -c` -eq 0 ]; then
+		describeInstance
+	fi
+	grep -B1 -A4 -m 1 $instanceName $describeInstanceFile > /tmp/instanceInfo_$instanceName 2>&1
+
+	#check if running or terminated
+	cat /tmp/instanceInfo_$instanceName | grep running > /tmp/instanceStatus_$instanceName
+	if [ `cat /tmp/instanceStatus_$instanceName | wc -c` -eq 0 ]; then
+		# not running
+		cat /tmp/instanceInfo_$instanceName | grep pending > /tmp/instanceStatus_$instanceName
+		if [ `cat /tmp/instanceStatus_$instanceName | wc -c` -ne 0 ]; then
+			describeInstance
+			echo "stopped"
+			exit 1
+		else
+			cat /tmp/instanceInfo_$instanceName | grep terminated > /tmp/instanceStatus_$instanceName
+			if [ `cat /tmp/instanceStatus_$instanceName | wc -c` -ne 0 ]; then
+				echo "terminated"
+				exit 1
+			else
+				cat /tmp/instanceInfo_$instanceName | grep shutting-down > /tmp/instanceStatus_$instanceName
+				if [ `cat /tmp/instanceStatus_$instanceName | wc -c` -ne 0 ]; then
+					echo "terminated"
+					exit 1
+				else
+					echo "stopped"
+					exit 1
+				fi
+			fi
+		fi
+	fi
+	
+	#running, get priviate IP Address
+	if [ "$subnet" == "unassigned" ]; then
+		IpAddr=`head -n 2 /tmp/instanceInfo_$instanceName | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $14}'`
+	else
+		IpAddr=`head -n 2 /tmp/instanceInfo_$instanceName | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $14}'`
+	fi
+
+	echo $IpAddr
+	exit 0
+}
+
+getType() {
+	#get local Instance ID
+	getInstancePrivate >/dev/null 2>&1
+	#get Type
+	if [ "$subnet" == "unassigned" ]; then
+		instanceType=`cat $describeInstanceFile | grep -m 1 $instance |  awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $9}'`
+		if [ "$instanceType" == "" ]; then
+			describeInstance
+		fi
+		instanceType=`cat $describeInstanceFile | grep -m 1 $instance |  awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $9}'`
+
+	else
+		instanceType=`cat $describeInstanceFile | grep -m 1 $instance |  awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $9}'`
+		if [ "$instanceType" == "" ]; then
+			describeInstance
+		fi
+		instanceType=`cat $describeInstanceFile | grep -m 1 $instance |  awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $9}'`
+
+	fi
+
+	echo $instanceType
+	return
+}
+
+getKey() {
+	#get local Instance ID
+	getInstancePrivate >/dev/null 2>&1
+	#get Key
+	if [ "$subnet" == "unassigned" ]; then
+		key=`cat $describeInstanceFile | grep -m 1 $instance |  awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $7}'`
+		if [ "$key" == "" ]; then
+			describeInstance
+		fi
+		key=`cat $describeInstanceFile | grep -m 1 $instance |  awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $7}'`
+
+	else
+		key=`cat $describeInstanceFile | grep -m 1 $instance |  awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $7}'`
+		if [ "$key" == "" ]; then
+			describeInstance
+		fi
+		key=`cat $describeInstanceFile | grep -m 1 $instance |  awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $7}'`
+
+	fi
+
+	echo $key
+	return
+}
+
+getAMI() {
+	#get local Instance ID
+	getInstancePrivate >/dev/null 2>&1
+	#get AMI
+	ami=`cat $describeInstanceFile | grep -m 1 $instance |  awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $3}'`
+	if [ "$ami" == "" ]; then
+		describeInstance
+	fi
+	ami=`cat $describeInstanceFile | grep -m 1 $instance |  awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $3}'`
+
+	echo $ami
+	return
+}
+
+getGroup() {
+	#get local Instance ID
+	getInstancePrivate >/dev/null 2>&1
+	#get group 
+	if [ "$subnet" == "unassigned" ]; then
+		group=`grep -B1 -A4 -m 1 $instance $describeInstanceFile |  grep -m 1 RESERVATION | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $4}'`
+		if [ "$group" == "" ]; then
+			describeInstance
+		fi
+		group=`grep -B1 -A4 -m 1 $instance $describeInstanceFile |  grep -m 1 RESERVATION | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $4}'`
+		if [ "$group" == "" ]; then
+			group=`grep -B1 -A4 -m 1 $instance $describeInstanceFile |  grep -m 1 INSTANCE | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $21}'`
+		fi
+	else
+		group=`grep -B1 -A6 -m 1 $instance $describeInstanceFile |  grep -m 1 GROUP | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'`
+		if [ "$group" == "" ]; then
+			describeInstance
+		fi
+		group=`grep -B1 -A6 -m 1 $instance $describeInstanceFile |  grep -m 1 GROUP | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'`
+		if [ "$group" == "" ]; then
+			group=`grep -B1 -A4 -m 1 $instance $describeInstanceFile |  grep -m 1 INSTANCE | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $21}'`
+		fi
+	fi
+
+	echo $group
+	return
+}
+
+getProfile() {
+	#get local Instance ID
+	getInstancePrivate >/dev/null 2>&1
+	#get Type
+	if [ "$subnet" == "unassigned" ]; then
+		instanceProfile=`cat $describeInstanceFile | grep -m 1 $instance |  awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $23}'`
+		if [ "$instanceProfile" == "" ]; then
+			describeInstance
+		fi
+		instanceProfile=`cat $describeInstanceFile | grep -m 1 $instance |  awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $23}'`
+
+	else
+		instanceProfile=`cat $describeInstanceFile | grep -m 1 $instance |  awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $22}'`
+		if [ "$instanceProfile" == "" ]; then
+			describeInstance
+		fi
+		instanceProfile=`cat $describeInstanceFile | grep -m 1 $instance |  awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $22}'`
+
+	fi
+
+	echo $instanceProfile
+	return
+}
+
+launchInstance() {
+	#get publickey
+	getKey >/dev/null 2>&1
+	if [ "$group" = "unassigned" ]; then
+		#get group
+		getGroup >/dev/null 2>&1
+	fi
+	#get AMI
+	getAMI >/dev/null 2>&1
+	#get Zone
+	getZone >/dev/null 2>&1
+	if [ "$instanceType" = "unassigned" ]; then
+		#get type
+		getType >/dev/null 2>&1
+	fi
+	#get AMI Profile
+	getProfile >/dev/null 2>&1
+
+	if [ "$subnet" == "unassigned" ]; then
+		#NOT VPC
+		if [ "$instanceProfile" = "" ] || [ "$instanceProfile" = "default" ]; then
+			newInstance=`ec2-run-instances -O $AmazonAccessKey -W $AmazonSecretKey -k $key -g $group -t $instanceType -z $zone --region $Region $ami | grep  -m 1 INSTANCE | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'`
+		else
+			newInstance=`ec2-run-instances -O $AmazonAccessKey -W $AmazonSecretKey -k $key -g $group -t $instanceType -z $zone -p $instanceProfile --region $Region $ami | grep  -m 1 INSTANCE | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'`
+		fi
+	else	# VPC
+		if [ "$instanceProfile" = "" ] || [ "$instanceProfile" = "default" ]; then
+			if [ "$group" != "default" ]; then
+				if [ "$IPaddress" = "autoassign" ]; then
+					newInstance=`ec2-run-instances -O $AmazonAccessKey -W $AmazonSecretKey -k $key -g $group -t $instanceType -z $zone  --region $Region -s $subnet $ami | grep  -m 1 INSTANCE | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'`
+				else
+					newInstance=`ec2-run-instances -O $AmazonAccessKey -W $AmazonSecretKey -k $key -g $group -t $instanceType -z $zone  --region $Region -s $subnet --private-ip-address $IPaddress $ami | grep  -m 1 INSTANCE | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'`
+				fi
+			else
+				if [ "$IPaddress" = "autoassign" ]; then
+					newInstance=`ec2-run-instances -O $AmazonAccessKey -W $AmazonSecretKey -k $key -t $instanceType -z $zone --region $Region -s $subnet $ami | grep  -m 1 INSTANCE | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'`
+				else
+					newInstance=`ec2-run-instances --O $AmazonAccessKey -W $AmazonSecretKey -k $key -t $instanceType -z $zone --region $Region -s $subnet --private-ip-address $IPaddress $ami | grep  -m 1 INSTANCE | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'`
+				fi
+			fi
+		else
+			if [ "$group" != "default" ]; then
+				if [ "$IPaddress" = "autoassign" ]; then
+					newInstance=`ec2-run-instances -O $AmazonAccessKey -W $AmazonSecretKey -k $key -g $group -t $instanceType -z $zone -p $instanceProfile --region $Region -s $subnet $ami | grep  -m 1 INSTANCE | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'`
+				else
+					newInstance=`ec2-run-instances -O $AmazonAccessKey -W $AmazonSecretKey -k $key -g $group -t $instanceType -z $zone -p $instanceProfile --region $Region -s $subnet --private-ip-address $IPaddress $ami | grep  -m 1 INSTANCE | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'`
+				fi
+			else
+				if [ "$IPaddress" = "autoassign" ]; then
+					newInstance=`ec2-run-instances -O $AmazonAccessKey -W $AmazonSecretKey -k $key -t $instanceType -z $zone -p $instanceProfile --region $Region -s $subnet $ami | grep  -m 1 INSTANCE | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'`
+				else
+					newInstance=`ec2-run-instances -O $AmazonAccessKey -W $AmazonSecretKey -k $key -t $instanceType -z $zone -p $instanceProfile --region $Region -s $subnet --private-ip-address $IPaddress $ami | grep  -m 1 INSTANCE | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'`
+				fi
+			fi
+		fi
+	fi
+	echo $newInstance
+	return
+}
+
+terminateInstance() {
+	#terminate Instance
+	ec2-terminate-instances -O $AmazonAccessKey -W $AmazonSecretKey --region $Region $instanceName > /tmp/termInstanceInfo_$instanceName 2>&1
+	return
+}
+
+stopInstance() {
+	#terminate Instance
+	ec2-stop-instances -O $AmazonAccessKey -W $AmazonSecretKey --region $Region $instanceName > /tmp/stopInstanceInfo_$instanceName 2>&1
+	return
+}
+
+startInstance() {
+	#terminate Instance
+	ec2-start-instances -O $AmazonAccessKey -W $AmazonSecretKey --region $Region $instanceName > /tmp/startInstanceInfo_$instanceName 2>&1
+
+	cat /tmp/startInstanceInfo_$instanceName | grep INSTANCE > /tmp/startInstanceStatus_$instanceName
+	if [ `cat /tmp/startInstanceStatus_$instanceName | wc -c` -eq 0 ]; then
+		echo "Failed, check /tmp/startInstanceInfo_$instanceName"
+		exit 1
+	fi
+	echo "Success"
+	exit 0
+}
+
+assignElasticIP() {
+	#terminate Instance
+
+        if [ "$subnet" == "unassigned" ]; then
+		ec2-associate-address -O $AmazonAccessKey -W $AmazonSecretKey -i $instanceName $IPAddress > /tmp/assignElasticIPInfo_$IPAddress 2>&1
+	else
+		EIP=`ec2-describe-addresses -O $AmazonAccessKey -W $AmazonSecretKey --region $Region $IPAddress | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $4}'`
+                ec2-associate-address -O $AmazonAccessKey -W $AmazonSecretKey --region $Region -i $instanceName -a $EIP > /tmp/assignElasticIPInfo_$IPAddress 2>&1
+	fi
+
+	cat /tmp/assignElasticIPInfo_$IPAddress | grep ADDRESS > /tmp/assignElasticIPStatus_$IPAddress
+	if [ `cat /tmp/assignElasticIPStatus_$IPAddress | wc -c` -eq 0 ]; then
+		echo "Failed, check /tmp/assignElasticIPInfo_$IPAddress"
+		exit 1
+	fi
+
+	echo "Success"
+	exit 0
+}
+
+deassignElasticIP() {
+	#terminate Instance
+	ec2-disassociate-address -O $AmazonAccessKey -W $AmazonSecretKey $IPAddress > /tmp/deassignElasticIPInfo_$IPAddress 2>&1
+
+	cat /tmp/deassignElasticIPInfo_$IPAddress | grep ADDRESS > /tmp/deassignElasticIPStatus_$IPAddress
+	if [ `cat /tmp/deassignElasticIPStatus_$IPAddress | wc -c` -eq 0 ]; then
+		echo "Failed, check /tmp/deassignElasticIPStatus_$IPAddress"
+		exit 1
+	fi
+
+	echo "Success"
+	exit 0
+}
+
+getSubnet() {
+        #get local Instance ID
+        getInstancePrivate >/dev/null 2>&1
+        #get Subnet
+        subnet=`cat $describeInstanceFile | grep -m 1 $instance |  awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $16}'`
+        if [ "$subnet" == "" ]; then
+                describeInstance
+        fi
+        subnet=`cat $describeInstanceFile | grep -m 1 $instance |  awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $16}'`
+
+	if [[ $subnet == *"subnet"* ]]
+	then
+        	echo $subnet
+	else
+		echo "failed"
+	fi
+
+        return
+}
+
+
+case "$1" in
+  getInstance)
+  	getInstance
+	;;
+  getZone)
+  	getZone
+	;;
+  getPrivateIP)
+  	getPrivateIP
+	;;
+  getKey)
+  	getKey
+	;;
+  getAMI)
+  	getAMI
+	;;
+  getType)
+  	getType
+	;;
+  launchInstance)
+  	launchInstance
+	;;
+  terminateInstance)
+  	terminateInstance
+	;;
+  stopInstance)
+  	stopInstance
+	;;
+  startInstance)
+  	startInstance
+	;;
+  assignElasticIP)
+  	assignElasticIP
+	;;
+  deassignElasticIP)
+  	deassignElasticIP
+	;;
+  getProfile)
+  	getProfile
+	;;
+  getGroup)
+  	getGroup
+	;;
+  getSubnet)
+  	getSubnet
+	;;
+  *)
+	echo $"Usage: $0 {launchInstance|getInstance|getZone|getPrivateIP|getType|getKey|getAMI|terminateInstance|startInstance|assignElasticIP|deassignElasticIP|getProfile|stopInstance|getGroup|getSubnet}"
+	exit 1
+esac
+
+exit $?

oam/cloud/MCSVolumeCmds.sh

@@ -0,0 +1,337 @@
+#! /bin/sh
+# MCSVolumeCmds.sh
+# describe, detach and attach Volume Storage from a Cloud environment
+#
+# 1. Amazon EC2
+
+prefix=/usr/local
+
+#check command
+if [ "$1" = "" ]; then
+	echo "Enter Command Name: {create|describe|detach|attach|delete|createTag}"
+	exit 1
+fi
+
+if [ "$1" = "create" ]; then
+	if [ "$2" = "" ]; then
+		echo "Enter size of the volume, in GiB (1-1024)"
+		exit 1
+	fi
+	volumeSize="$2"
+
+	#get module-type
+	if [ "$3" = "" ]; then
+		echo "Enter Module Type"
+		exit 1
+	fi
+	moduleType="$3"
+fi
+
+if [ "$1" = "describe" ]; then
+	if [ "$2" = "" ]; then
+		echo "Enter Volume Name"
+		exit 1
+	fi
+	volumeName="$2"
+fi
+
+if [ "$1" = "detach" ]; then
+	if [ "$2" = "" ]; then
+		echo "Enter Volume Name"
+		exit 1
+	fi
+	volumeName="$2"
+fi
+
+if [ "$1" = "attach" ]; then
+	if [ "$2" = "" ]; then
+		echo "Enter Volume Name"
+		exit 1
+	fi
+	volumeName="$2"
+
+	#get instance-name and device-name
+	if [ "$3" = "" ]; then
+		echo "Enter Instance Name"
+		exit 1
+	fi
+	instanceName="$3"
+
+	if [ "$4" = "" ]; then
+		echo "Enter Device Name"
+		exit 1
+	fi
+	deviceName="$4"
+fi
+
+if [ "$1" = "delete" ]; then
+	if [ "$2" = "" ]; then
+		echo "Enter Volume Name"
+		exit 1
+	fi
+	volumeName="$2"
+fi
+
+if [ "$1" = "createTag" ]; then
+	if [ "$2" = "" ]; then
+		echo "Enter Resource Name"
+		exit 1
+	fi
+	resourceName="$2"
+
+	if [ "$3" = "" ]; then
+		echo "Enter Tag Name"
+		exit 1
+	fi
+	tagName="$3"
+
+	if [ "$4" = "" ]; then
+		echo "Enter Tag Value"
+		exit 1
+	fi
+	tagValue="$4"
+fi
+
+
+test -f $prefix/mariadb/columnstore/post/functions && . $prefix/mariadb/columnstore/post/functions
+
+ec2=`$prefix/mariadb/columnstore/bin/getConfig Installation EC2_HOME`
+
+if [ $ec2 == "unassigned" ]; then
+       STATUS="unknown"
+        RETVAL=1
+fi
+
+java=`$prefix/mariadb/columnstore/bin/getConfig Installation JAVA_HOME`
+path=`$prefix/mariadb/columnstore/bin/getConfig Installation EC2_PATH`
+
+export PATH=$path
+export EC2_HOME=$ec2
+export JAVA_HOME=$java
+
+# get Keys and region
+AmazonAccessKeyFile=`$prefix/mariadb/columnstore/bin/getConfig Installation AmazonAccessKey`
+if [ $AmazonAccessKeyFile == "unassigned" ]; then
+	echo "FAILED: missing Config Setting AmazonAccessKey : $AmazonAccessKeyfile"
+	exit 1
+fi
+
+AmazonSecretKeyFile=`$prefix/mariadb/columnstore/bin/getConfig Installation AmazonSecretKey`
+if [ $AmazonSecretKeyFile == "unassigned" ]; then
+	echo "FAILED: missing Config Setting AmazonSecretKeyFile : $AmazonSecretKeyFile"
+	exit 1
+fi
+
+AmazonAccessKey=`cat $AmazonAccessKeyFile`
+AmazonSecretKey=`cat $AmazonSecretKeyFile`
+
+if test ! -f $AmazonAccessKeyfile ; then
+	echo "FAILED: missing AmazonAccessKeyfile : $AmazonAccessKeyfile"
+	exit 1
+fi
+
+if test ! -f $AmazonSecretKeyfile ; then
+	echo "FAILED: missing AmazonSecretKeyfile : $AmazonSecretKeyfile"
+	exit 1
+fi
+
+Region=`$prefix/mariadb/columnstore/bin/getConfig Installation AmazonRegion`
+
+
+checkInfostatus() {
+	#check if attached
+	cat /tmp/volumeInfo_$volumeName | grep attached > /tmp/volumeStatus_$volumeName
+	if [ `cat /tmp/volumeStatus_$volumeName | wc -c` -ne 0 ]; then
+		STATUS="attached"
+		RETVAL=0
+		return
+	fi
+	#check if available
+	cat /tmp/volumeInfo_$volumeName | grep available > /tmp/volumeStatus_$volumeName
+	if [ `cat /tmp/volumeStatus_$volumeName | wc -c` -ne 0 ]; then
+		STATUS="available"
+		RETVAL=0
+		return
+	fi
+	#check if detaching
+	cat /tmp/volumeInfo_$volumeName | grep detaching > /tmp/volumeStatus_$volumeName
+	if [ `cat /tmp/volumeStatus_$volumeName | wc -c` -ne 0 ]; then
+		STATUS="detaching"
+		RETVAL=0
+		return
+	fi
+	#check if attaching
+	cat /tmp/volumeInfo_$volumeName | grep attaching > /tmp/volumeStatus_$volumeName
+	if [ `cat /tmp/volumeStatus_$volumeName | wc -c` -ne 0 ]; then
+		STATUS="attaching"
+		RETVAL=0
+		return
+	fi
+	#check if doesn't exist
+	cat /tmp/volumeInfo_$volumeName | grep "does not exist" > /tmp/volumeStatus_$volumeName
+	if [ `cat /tmp/volumeStatus_$volumeName | wc -c` -ne 0 ]; then
+		STATUS="does-not-exist"
+		RETVAL=1
+		return
+	fi
+	#check if reports already attached from attach command
+	cat /tmp/volumeInfo_$volumeName | grep "already attached" > /tmp/volumeStatus_$volumeName
+	if [ `cat /tmp/volumeStatus_$volumeName | wc -c` -ne 0 ]; then
+		STATUS="already-attached"
+		RETVAL=1
+		return
+	fi
+	#any other, unknown error
+	STATUS="unknown"
+	RETVAL=1
+	return
+}
+
+createvolume() {
+	# get zone
+	zone=`$prefix/mariadb/columnstore/bin/MCSInstanceCmds.sh getZone`
+
+	if [ $moduleType == "um" ]; then
+		# get type
+		volumeType=`$prefix/mariadb/columnstore/bin/getConfig Installation UMVolumeType`
+		if [ $volumeType == "io1" ]; then
+			# get IOPS
+			volumeIOPS=`$prefix/mariadb/columnstore/bin/getConfig Installation UMVolumeIOPS`
+		fi
+	else	# pm
+		# get type
+		volumeType=`$prefix/mariadb/columnstore/bin/getConfig Installation PMVolumeType`
+		if [ $volumeType == "io1" ]; then
+			# get IOPS
+			volumeIOPS=`$prefix/mariadb/columnstore/bin/getConfig Installation PMVolumeIOPS`
+		fi
+	fi
+
+	#create volume
+	if [ $volumeType == "io1" ]; then
+		volume=`ec2-create-volume -O $AmazonAccessKey -W $AmazonSecretKey --region $Region -z $zone -s $volumeSize -t $volumeType -iops $volumeIOPS  | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'`
+	else
+		volume=`ec2-create-volume -O $AmazonAccessKey -W $AmazonSecretKey --region $Region -z $zone -s $volumeSize -t $volumeType | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'`
+	fi
+
+	echo $volume
+	return
+}
+
+describevolume() {
+	#describe volume
+	ec2-describe-volumes -O $AmazonAccessKey -W $AmazonSecretKey --region $Region $volumeName > /tmp/volumeInfo_$volumeName 2>&1
+
+	checkInfostatus
+	echo $STATUS
+	return
+}
+
+detachvolume() {
+	#detach volume
+	ec2-detach-volume -O $AmazonAccessKey -W $AmazonSecretKey --region $Region $volumeName > /tmp/volumeInfo_$volumeName 2>&1
+
+	checkInfostatus
+	if [ $STATUS == "detaching" ]; then
+		retries=1
+		while [ $retries -ne 60 ]; do
+			#retry until it's attached
+			ec2-detach-volume -O $AmazonAccessKey -W $AmazonSecretKey --region $Region $volumeName > /tmp/volumeInfo_$volumeName 2>&1
+		
+			checkInfostatus
+			if [ $STATUS == "available" ]; then
+				echo "available"
+				exit 0
+			fi
+			((retries++))
+			sleep 1
+		done
+		test -f /usr/local/mariadb/columnstore/post/functions && . /usr/local/mariadb/columnstore/post/functions
+		$prefix/mariadb/columnstore/bin/cplogger -w 100 "detachvolume failed: $STATUS"
+		echo "failed"
+		exit 1
+	fi
+
+	if [ $STATUS == "available" ]; then
+		echo "available"
+		exit 0
+	fi
+
+	test -f /usr/local/mariadb/columnstore/post/functions && . /usr/local/mariadb/columnstore/post/functions
+	$prefix/mariadb/columnstore/bin/cplogger -w 100 "detachvolume failed status: $STATUS"
+	echo $STATUS
+	exit 1
+}
+
+attachvolume() {
+
+	#detach volume
+	ec2-attach-volume -O $AmazonAccessKey -W $AmazonSecretKey --region $Region $volumeName -i $instanceName -d $deviceName > /tmp/volumeInfo_$volumeName 2>&1
+
+	checkInfostatus
+	if [ $STATUS == "attaching" -o $STATUS == "already-attached" ]; then
+		retries=1
+		while [ $retries -ne 60 ]; do
+			#check status until it's attached
+			describevolume
+			if [ $STATUS == "attached" ]; then
+				echo "attached"
+				exit 0
+			fi
+			((retries++))
+			sleep 1
+		done
+		test -f /usr/local/mariadb/columnstore/post/functions && . /usr/local/mariadb/columnstore/post/functions
+		$prefix/mariadb/columnstore/bin/cplogger -w 100 "attachvolume failed: $STATUS"
+		echo "failed"
+		exit 1
+	fi
+
+	if [ $STATUS == "attached" ]; then
+		echo "attached"
+		exit 0
+	fi
+
+	test -f /usr/local/mariadb/columnstore/post/functions && . /usr/local/mariadb/columnstore/post/functions
+	$prefix/mariadb/columnstore/bin/cplogger -w 100 "attachvolume failed: $STATUS"
+	echo $STATUS
+	exit 1
+}
+
+deletevolume() {
+	#delete volume
+	ec2-delete-volume -O $AmazonAccessKey -W $AmazonSecretKey --region $Region $volumeName > /tmp/deletevolume_$volumeName 2>&1
+	return
+}
+
+createTag() {
+	#create tag
+	ec2-create-tags -O $AmazonAccessKey -W $AmazonSecretKey --region $Region $resourceName --tag $tagName=$tagValue > /tmp/createTag_$volumeName 2>&1
+	return
+}
+
+case "$1" in
+  create)
+  	createvolume
+	;;
+  describe)
+  	describevolume
+	;;
+  detach)
+  	detachvolume
+	;;
+  attach)
+  	attachvolume
+	;;
+  delete)
+  	deletevolume
+	;;
+  createTag)
+  	createTag
+	;;
+  *)
+	echo $"Usage: $0 {create|describe|detach|attach|delete|}"
+	exit 1
+esac
+
+exit $?

oam/etc/ConsoleCmds.xml

@@ -34,7 +34,12 @@
 		<Arg4>STATUS to to view statistics and progress</Arg4>
 	</Cmd4>
 	<Cmd5>
-                <Name>AVAILABLE</Name>
+		<Name>findObjectFile</Name>
+		<Desc1>Get the name of the directory containing the first file of the object</Desc1>
+		<Arg1>Object ID (OID) of object</Arg1>
+		<Arg2>Schema</Arg2>
+		<Arg3>Table</Arg3>
+		<Arg4>Column</Arg4>
 	</Cmd5>
 	<Cmd6>
 		<Name>getModuleTypeConfig</Name>

oam/install_scripts/binary_installer.sh

@@ -234,7 +234,7 @@ if { $INSTALLTYPE == "initial"} {
 	send_user "Run Module Installer                            "
 	send " \n"
 	send date\n
-	send "ssh $USERNAME@$SERVER '$INSTALLDIR/bin/module_installer.sh --module=$MODULETYPE --port=$MYSQLPORT --installdir=$INSTALLDIR $MODULETYPE'\n"
+	send "ssh $USERNAME@$SERVER '$INSTALLDIR/bin/module_installer.sh --module=$MODULETYPE --port=$MYSQLPORT --installdir=$INSTALLDIR'\n"
 	set timeout 10
 	expect {
 		"word: " { send "$PASSWORD\n" }

oam/install_scripts/columnstore

@@ -57,8 +57,10 @@ if [ "x$has_um" = x ]; then
 fi
 
 user=$USER
+sudo=sudo
 if [ -z "$user" ]; then
         user=root
+        sudo=" "
 fi
 
 checkInstallSetup() {
@@ -66,7 +68,7 @@ checkInstallSetup() {
 	if [ $InitialInstallFlag != "y" ]; then
 		echo "Please run the postConfigure install script, check the Installation Guide"
 		echo "for additional details"
-		rm -f /var/lock/subsys/columnstore
+		sudo rm -f /var/lock/subsys/columnstore
 		exit 1
 	fi
 }
@@ -79,13 +81,13 @@ start() {
 		exit 0
 	fi
 
-	(mkdir -p /var/lock/subsys && touch /var/lock/subsys/columnstore) >/dev/null 2>&1
+	(sudo mkdir -p /var/lock/subsys && touch /var/lock/subsys/columnstore) >/dev/null 2>&1
 
 	if [ -x $InstallDir/bin/columnstore.pre-start ]; then
 		$InstallDir/bin/columnstore.pre-start
 		if [ $? -ne 0 ]; then
 			echo "Error running MariaDB Columnstore pre-start script, not starting MariaDB Columnstore"
-			rm -f /var/lock/subsys/columnstore
+			sudo rm -f /var/lock/subsys/columnstore
 			exit 1
 		fi
 	fi
@@ -106,21 +108,21 @@ start() {
 
 	RETVAL=0
 	echo "Starting MariaDB Columnstore Database Platform"
-	rm -f /tmp/StopColumnstore
+	sudo rm -f /tmp/StopColumnstore
 	exec $InstallDir/bin/run.sh $InstallDir/bin/ProcMon  > /dev/null 2>&1 &
 
 	return $RETVAL
 }	
 stop() {
 	echo "Shutting down MariaDB Columnstore Database Platform"
-	touch /tmp/StopColumnstore
+	sudo touch /tmp/StopColumnstore
-	pkill ProcMon
+	sudo pkill ProcMon
-	pkill ProcMgr
+	sudo pkill ProcMgr
 	sleep 1
 	$InstallDir/bin/clearShm
 	RETVAL=$?
-	rm -f /var/lock/subsys/columnstore
+	sudo rm -f /var/lock/subsys/columnstore
-	fuser -k 8604/tcp > /dev/null 2>&1
+	sudo fuser -k 8604/tcp > /dev/null 2>&1
 	test -f $InstallDir/mysql/mysql-Columnstore || return $RETVAL
 	$InstallDir/mysql/mysql-Columnstore stop > /dev/null 2>&1
 	if [ -x $InstallDir/bin/columnstore.post-stop ]; then

oam/install_scripts/columnstoreAlias

@@ -3,6 +3,7 @@
 alias mcsmysql='/usr/local/mariadb/columnstore/mysql/bin/mysql --defaults-file=/usr/local/mariadb/columnstore/mysql/my.cnf -u root'
 alias ma=/usr/local/mariadb/columnstore/bin/mcsadmin
 alias mcsadmin=/usr/local/mariadb/columnstore/bin/mcsadmin
+alias cpimport=/usr/local/mariadb/columnstore/bin/cpimport
 alias home='cd /usr/local/mariadb/columnstore'
 alias log='cd /var/log/mariadb/columnstore/'
 alias core='cd /var/log/mariadb/columnstore/corefiles'

oam/install_scripts/columnstoreSyslog7

@@ -1,4 +1,4 @@
-# MariaDb Columnstore Database Platform Logging
+# MariaDB Columnstore Database Platform Logging
 local1.crit -/var/log/mariadb/columnstore/crit.log
 local1.err -/var/log/mariadb/columnstore/err.log
 local1.warning -/var/log/mariadb/columnstore/warning.log

oam/install_scripts/module_installer.sh

@@ -113,18 +113,6 @@ if [ $EUID -eq 0 -a -f $COLUMNSTORE_INSTALL_DIR/local/rc.local.columnstore ]; th
 	fi
 fi
 
-if [ $user != "root" ]; then
-	echo "Setup .bashrc on Module for non-root"
-
-	eval userhome=~$user
-	bashFile=$userhome/.bashrc
-	touch ${bashFile}
-
-	echo " " >> ${bashFile}
-	echo "export COLUMNSTORE_INSTALL_DIR=$COLUMNSTORE_INSTALL_DIR" >> ${bashFile}
-	echo "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$COLUMNSTORE_INSTALL_DIR/lib:$COLUMNSTORE_INSTALL_DIR/mysql/lib/mysql" >> ${bashFile}
-fi
-
 plugin=`$COLUMNSTORE_INSTALL_DIR/bin/getConfig SystemConfig DataFilePlugin`
 if [ -n "$plugin" ]; then
 	echo "Setup .bashrc on Module for local-query"

oam/install_scripts/post-install

@@ -8,9 +8,12 @@ prefix=/usr/local
 installdir=$prefix/mariadb/columnstore
 rpmmode=install
 user=$USER
+sudo=sudo
 if [ -z "$user" ]; then
-	user=root
+        user=root
+        sudo=" "
 fi
+
 quiet=0
 
 for arg in "$@"; do
@@ -81,27 +84,52 @@ fi
 # softlink for libperl.sp, used by cplogger
 sudo ln -s /usr/lib64/perl5/CORE/libperl.so /usr/lib64/libperl.so >/dev/null 2>&1
 
-# TODO-based on early experience with HDP it doesn't install bigtop
+#setup .bashrc
-# so the code below will not correctly detect JAVA_HOME.  Need to 
+if [ $user = "root" ]; then
-# decide how to handle
+	bashFile=/root/.bashrc
-# Autodetect JAVA_HOME if not defined
+else
-#if [ -e /usr/libexec/bigtop-detect-javahome ]; then
+	bashFile=$HOME/.bashrc
-#. /usr/libexec/bigtop-detect-javahome
+fi
-#elif [ -e /usr/lib/bigtop-utils/bigtop-detect-javahome ]; then
+
-#. /usr/lib/bigtop-utils/bigtop-detect-javahome
+touch ${bashFile}
-#fi
+/bin/cp -f ${bashFile} ${bashFile}.columnstoreSave > /dev/null 2>&1
-#ln -sf $JAVA_HOME/jre/lib/amd64/server/libjvm.so
+
-#chown -h $user.$user libjvm.so >/dev/null 2>&1
+if [ $user != "root" ]; then
+
+	egrep -qs 'MariaDB Columnstore Non-Root' ${bashFile}
+	rc1=$?
+	if [ $rc1 -ne 0 ]; then
+        	echo " " >> ${bashFile}
+		echo "# MariaDB Columnstore Non-Root Environment Variables" >> ${bashFile}
+        	echo "export COLUMNSTORE_INSTALL_DIR=$COLUMNSTORE_INSTALL_DIR" >> ${bashFile}
+        	echo "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$COLUMNSTORE_INSTALL_DIR/lib:$COLUMNSTORE_INSTALL_DIR/mysql/lib/mysql" >> ${bashFile}
+        	. ${bashFile}
+	fi
+fi
+
+if [ $installdir != "/usr/local/mariadb/columnstore" ]; then
+        sed -i -e s@/usr/local/mariadb/columnstore@$installdir@g $installdir/bin/columnstoreAlias
+fi
+egrep -qs 'MariaDB Columnstore Database Platform Alias Commands' ${bashFile}
+rc1=$?
+egrep -qs 'MariaDB Columnstore Alias Commands' ${bashFile}
+rc2=$?
+if [ $rc1 -ne 0 -a $rc2 -ne 0 ]; then
+        cat $installdir/bin/columnstoreAlias >> ${bashFile}
+        echo " " >> ${bashFile}
+fi
 
 cd /
 
-test -d /var/log/mariadb/columnstore || mkdir /var/log/mariadb/columnstore >/dev/null 2>&1
+test -d /var/log/mariadb || sudo mkdir /var/log/mariadb >/dev/null 2>&1
+test -d /var/log/mariadb/columnstore || sudo mkdir /var/log/mariadb/columnstore >/dev/null 2>&1
+sudo chmod -R 777 /var/log/mariadb
+
 test -d /var/log/mariadb/columnstore/archive || mkdir /var/log/mariadb/columnstore/archive >/dev/null 2>&1
 test -d /var/log/mariadb/columnstore/corefiles || mkdir /var/log/mariadb/columnstore/corefiles >/dev/null 2>&1
 test -d /var/log/mariadb/columnstore/trace || mkdir /var/log/mariadb/columnstore/trace >/dev/null 2>&1
 test -h /var/log/mariadb/columnstore/data && rm -f /var/log/mariadb/columnstore/data
 # make sure trace dir is world-writable and sticky
-chmod 1755 /var/log/mariadb/columnstore/trace >/dev/null 2>&1
 test -d $installdir/data || mkdir $installdir/data
 test -d $installdir/data1 || mkdir $installdir/data1
 test -d $installdir/data1/systemFiles || mkdir $installdir/data1/systemFiles
@@ -180,16 +208,20 @@ if [ $user = "root" ]; then
 	chmod 777 -R /dev/shm
 	rm -f /etc/default/columnstore
 else
+	sudo chmod 777 /tmp
+	sudo rm -fr /tmp/* > /dev/null 2>&1
 	sudo $installdir/bin/syslogSetup.sh --installdir=$installdir install > /tmp/syslog_install.log 2>&1
 	sudo chown $user:$user $installdir/etc/Columnstore.xml
-	sudo chmod 755 /var/log
-	sudo chmod -R 755 /var/log/mariadb/columnstore
 	sudo chmod -R 777 /dev/shm
+	sudo chmod 777 /var/lock/subsys
 	sudo rm -f /var/lock/subsys/mysql-Columnstore
-	sudo rm -f /tmp/StopColumnstore
+	sudo chmod 777 /etc/fstab	
 
 	sed -i -e s@/usr/local/mariadb/columnstore@$installdir@g $installdir/bin/columnstore.def
 	sudo cp $installdir/bin/columnstore.def /etc/default/columnstore
+
+	sed -i -e s@prefix=/usr/local@prefix=$HOME@g $installdir/bin/*
+
 fi
 
 #check if MariaDB Columnstore system logging was setup

oam/install_scripts/post-mysqld-install

@@ -82,9 +82,17 @@ chown -R $user.$user $installdir/mysql
 if [ -f $installdir/lib/libcalmysql.so.1.0.0 ]; then
 	libcalmysql=$installdir/lib/libcalmysql.so.1.0.0
 	libudfsdk=$installdir/lib/libudf_mysql.so.1.0.0
+    is_columnstore_tables=$installdir/lib/is_columnstore_tables.so.1.0.0
+    is_columnstore_columns=$installdir/lib/is_columnstore_columns.so.1.0.0
+    is_columnstore_extents=$installdir/lib/is_columnstore_extents.so.1.0.0
+    is_columnstore_files=$installdir/lib/is_columnstore_files.so.1.0.0
 elif [ -f $installdir/lib/libcalmysql.so.1 ]; then
 	libcalmysql=$installdir/lib/libcalmysql.so.1
 	libudfsdk=$installdir/lib/libudf_mysql.so.1
+    is_columnstore_tables=$installdir/lib/is_columnstore_tables.so.1
+    is_columnstore_columns=$installdir/lib/is_columnstore_columns.so.1
+    is_columnstore_extents=$installdir/lib/is_columnstore_extents.so.1
+    is_columnstore_files=$installdir/lib/is_columnstore_files.so.1
 else
 	libcalmysql=
 fi
@@ -93,8 +101,12 @@ if [ -n "$libcalmysql" ]; then
 
 	cd $installdir/mysql/lib/plugin
 	ln -sf $libcalmysql libcalmysql.so
-        ln -sf $libcalmysql libcalmysqlent.so
+    ln -sf $libcalmysql libcalmysqlent.so
 	ln -sf $libudfsdk libudf_mysql.so
+    ln -sf $is_columnstore_tables is_columnstore_tables.so
+    ln -sf $is_columnstore_columns is_columnstore_columns.so
+    ln -sf $is_columnstore_extents is_columnstore_extents.so
+    ln -sf $is_columnstore_files is_columnstore_files.so
 fi
 
 # cleanup previous install mysql replication files

oam/install_scripts/pre-uninstall

@@ -33,22 +33,27 @@ for arg in "$@"; do
 	fi
 done
 
+if [ $user = "root" ]; then
+	. /root/.bashrc
+else
+	. $HOME/.bashrc
+fi
+
 #stop services
 $installdir/bin/columnstore stop > /dev/null 2>&1
 $installdir/myql/columnstore-Mysql stop > /dev/null 2>&1
 
 # uninstall OS scripts updated by postConfigure
-if test -f /root/.bashrc.columnstoreSave ; then
-	$sudo mv -f /root/.bashrc.columnstoreSave /root/.bashrc > /dev/null 2>&1
-fi
 if test -f /etc/exports.columnstoreSave ; then
 	$sudo mv -f /etc/exports.columnstoreSave /etc/exports > /dev/null 2>&1
 else
 	$sudo rm -f /etc/exports > /dev/null 2>&1
 fi
-if test -f /etc/rc.d/rc.local.columnstoreSave ; then
+
+if test -f /etc/rc.d/rc.local.columnstoreSave ; then	
 	$sudo mv -f /etc/rc.d/rc.local.columnstoreSave /etc/rc.d/rc.local > /dev/null 2>&1
 fi
+
 if test -f /etc/rc.local.columnstoreSave ; then
 	$sudo mv -f /etc/rc.local.columnstoreSave /etc/rc.local > /dev/null 2>&1
 fi
@@ -65,23 +70,23 @@ fi
 #remove log file directories
 #rm -rf /var/log/mariadb/columnstore > /dev/null 2>&1
 #rm -f $installdir/mysql/db/*.err > /dev/null 2>&1
-rm -f /var/log/mariadb/columnstore/activeAlarms > /dev/null 2>&1
+sudo rm -f /var/log/mariadb/columnstore/activeAlarms > /dev/null 2>&1
-rm -f /var/log/mariadb/columnstore/*.log1 > /dev/null 2>&1
+sudo rm -f /var/log/mariadb/columnstore/*.log1 > /dev/null 2>&1
 rm -rf $installdir/mysql/db/columnstore_log_archive > /dev/null 2>&1
 sudo rm -f /etc/default/columnstore
 
 # remove Mariab Columnstore Log Rotate File abd Transaction Log File
-rm -f /etc/logrotate.d/columnstore > /dev/null 2>&1
+sudo rm -f /etc/logrotate.d/columnstore > /dev/null 2>&1
-rm -f /etc/cron.d/transactionLog > /dev/null 2>&1
+sudo rm -f /etc/cron.d/transactionLog > /dev/null 2>&1
 
 # delete Mariab Columnstore shared memory segments
 $installdir/bin/clearShm  > /dev/null 2>&1
 
 # delete prat files
-rm -f /etc/cron.d/ps > /dev/null 2>&1
+sudo rm -f /etc/cron.d/ps > /dev/null 2>&1
-rm -f /etc/pscollect > /dev/null 2>&1
+sudo rm -f /etc/pscollect > /dev/null 2>&1
-/etc/init.d/crond reload  > /dev/null 2>&1
+sudo /etc/init.d/crond reload  > /dev/null 2>&1
-sudo systemctl reload crond.service > /dev/null 2>&1
+sudo sudo systemctl reload crond.service > /dev/null 2>&1
 
 # delete tmp files
 rm -f $installdir/local/*.columnstore
@@ -90,7 +95,7 @@ rm -rf /tmp/bucketreuse
 rm -f /tmp/columnstore.txt
 rm -f /tmp/dbbuilder.*
 rm -f /tmp/dbrmfiles
-rm -f /var/lock/subsys/columnstore
+sudo rm -f /var/lock/subsys/columnstore
 rm -f /tmp/pkgcheck
 rm -f /tmp/upgrade-status.log.*
 rm -f /tmp/mount.log
@@ -122,8 +127,8 @@ systemctl=`which systemctl 2>/dev/null`
 if [ -n "$systemctl" ]; then
 
 	sudo systemctl disable columnstore >/dev/null 2>&1
-	rm -f /usr/lib/systemd/system/columnstore.service
+	sudo rm -f /usr/lib/systemd/system/columnstore.service
-	rm -f /lib/systemd/system/columnstore.service
+	sudo rm -f /lib/systemd/system/columnstore.service
 else
 	chkconfig=`which chkconfig 2>/dev/null`
 	if [ -n "$chkconfig" ]; then
@@ -158,9 +163,20 @@ fi
 #remove library paths
 if [ $installdir == "/usr/local/mariadb/columnstore" ]; then
         # remove library config file
-        rm -f /etc/ld.so.conf.d/columnstore.conf
+        sudo rm -f /etc/ld.so.conf.d/columnstore.conf
-        rm -f /etc/ld.so.cache
+        sudo rm -f /etc/ld.so.cache
-        ldconfig
+        sudo ldconfig
+fi
+
+#cleanup .bashrc	
+if [ $user = "root" ]; then
+        if test -f /root/.bashrc.columnstoreSave ; then
+                mv -f /root/.bashrc.columnstoreSave /root/.bashrc > /dev/null 2>&1
+        fi
+else
+        if test -f $HOME/.bashrc.columnstoreSave ; then
+                mv -f /home/$user/.bashrc.columnstoreSave /home/$user/.bashrc > /dev/null 2>&1
+        fi
 fi
 
 #tell user to run post configure script

oam/install_scripts/rsync.sh

@@ -6,7 +6,6 @@
 # Argument 1 - Remote Server Host Name or IP address
 # Argument 2 - Root Password of remote server
 # Argument 3 - Debug flag 1 for on, 0 for off
-set USERNAME "root@"
 set SERVER [lindex $argv 0]
 set PASSWORD [lindex $argv 1]
 set INSTALLDIR [lindex $argv 2]
@@ -15,13 +14,19 @@ set DEBUG [lindex $argv 3]
 log_user $DEBUG
 spawn -noecho /bin/bash
 
+if {[info exists env(USER)]} {
+    set USERNAME $env(USER)
+} else {
+    set USERNAME "root"
+}
+
 
 if { $PASSWORD == "ssh" } {
 	set USERNAME ""
 	set PASSWORD ""
 }
 
-set COMMAND "rsync -vopgr -e ssh --exclude=mysql/ --exclude=test/ --exclude=infinidb_vtable/ --exclude=infinidb_querystats/ --exclude=calpontsys/ --include=*/ --include=*/* --exclude=* $INSTALLDIR/mysql/db/ $USERNAME$SERVER:$INSTALLDIR/mysql/db/"
+set COMMAND "rsync -vopgr -e ssh --exclude=mysql/ --exclude=test/ --exclude=infinidb_vtable/ --exclude=infinidb_querystats/ --exclude=calpontsys/ --include=*/ --include=*/* --exclude=* $INSTALLDIR/mysql/db/ $USERNAME@$SERVER:$INSTALLDIR/mysql/db/"
 
 #
 # run command

oam/install_scripts/slave-rep-columnstore.sh

@@ -74,7 +74,6 @@ CHANGE MASTER TO
     	MASTER_PORT=$port,
    	MASTER_LOG_FILE='$masterlogfile',
     	MASTER_LOG_POS=$masterlogpos;
-	MASTER_USE_GTID=$masterlogpos;
 
 EOD
 

oam/install_scripts/syslogSetup.sh

@@ -5,13 +5,21 @@
 # syslogSetup.sh - install / uninstall MariaDB Columnstore system logging configuration
 
 # no point in going any further if not root... (only works in bash)
-test $EUID -eq 0 || exit 0
+#test $EUID -eq 0 || exit 0
 
 prefix=/usr/local
 installdir=$prefix/mariadb/columnstore
 syslog_conf=nofile
 rsyslog7=0
 
+user=$USER
+sudo=sudo
+if [ -z "$user" ]; then
+        user=root
+        sudo=" "
+fi
+
+
 for arg in "$@"; do
 	if [ `expr -- "$arg" : '--prefix='` -eq 9 ]; then
 		prefix="`echo $arg | awk -F= '{print $2}'`"
@@ -62,13 +70,13 @@ if [ "$daemon" = "nodaemon" ]; then
 
 	if [ -f /etc/syslog.conf ]; then
 		daemon="syslog"
-		/etc/init.d/syslog start > /dev/null 2>&1
+		sudo /etc/init.d/syslog start > /dev/null 2>&1
 	elif [ -f /etc/rsyslog.conf ]; then
 		daemon="rsyslog"
-		/etc/init.d/rsyslog start > /dev/null 2>&1
+		sudo /etc/init.d/rsyslog start > /dev/null 2>&1
 	elif [ -f /etc/init.d/syslog-ng ]; then
 		daemon="syslog-ng"
-		/etc/init.d/syslog-ng start > /dev/null 2>&1
+		sudo /etc/init.d/syslog-ng start > /dev/null 2>&1
 	fi
 fi
 
@@ -92,7 +100,7 @@ if [ "$daemon" = "syslog-ng" ]; then
 	fi
 elif [ "$daemon" = "rsyslog" ]; then
 	#check if rsyslog version 7 or greater
-	rsyslogd -v > /tmp/rsyslog.ver
+	sudo rsyslogd -v > /tmp/rsyslog.ver
 	cnt=`grep "rsyslogd 7" /tmp/rsyslog.ver | wc -l`
 	if [ $cnt -gt 0 ]; then
 		rsyslog7=1
@@ -149,30 +157,33 @@ checkSyslog
 if [ ! -z "$syslog_conf" ] ; then
 	$installdir/bin/setConfig -d Installation SystemLogConfigFile ${syslog_conf} >/dev/null 2>&1
 	if [ "$syslog_conf" != /etc/rsyslog.d/columnstore.conf ]; then
-		rm -f ${syslog_conf}.columnstoreSave
+		sudo rm -f ${syslog_conf}.columnstoreSave
-		cp ${syslog_conf} ${syslog_conf}.columnstoreSave >/dev/null 2>&1
+		sudo cp ${syslog_conf} ${syslog_conf}.columnstoreSave >/dev/null 2>&1
-		sed -i '/# MariaDB/,$d' ${syslog_conf}.columnstoreSave > /dev/null 2>&1
+		sudo sed -i '/# MariaDB/,$d' ${syslog_conf}.columnstoreSave > /dev/null 2>&1
 	fi
 
 	egrep -qs 'MariaDB Columnstore Database Platform Logging' ${syslog_conf}
 	if [ $? -ne 0 ]; then
-		#set the syslog for calpont logging
+		#set the syslog for ColumnStore logging
 		# remove older version incase it was installed by previous build
-		rm -rf /etc/rsyslog.d/columnstore.conf
+		sudo rm -rf /etc/rsyslog.d/columnstore.conf
 		if [ $rsyslog7 == 1 ]; then
-			rm -f /etc/rsyslog.d/49-columnstore.conf
+			sudo rm -f /etc/rsyslog.d/49-columnstore.conf
-			cat  ${columnstoreSyslogFile7} >> ${syslog_conf}
+			sudo cp  ${columnstoreSyslogFile7} ${syslog_conf}
-			chown syslog:adm /var/log/mariadb/columnstore
+			sudo chown syslog:adm /var/log/mariadb/columnstore >/dev/null 2>&1
 		else
-			cat  ${columnstoreSyslogFile} >> ${syslog_conf}
+			sudo cp  ${columnstoreSyslogFile} ${syslog_conf}
 		fi
 	fi
 
-	pkill -hup syslogd > /dev/null 2>&1
+	sudo etc/init.d/rsyslog restart  > /dev/null 2>&1
-	pkill -hup syslog-ng  > /dev/null 2>&1
+	sudo /etc/init.d/syslog restart  > /dev/null 2>&1
-	pkill -hup rsyslogd  > /dev/null 2>&1
+        sudo /etc/init.d/syslog-ng restart  > /dev/null 2>&1
-	/etc/init.d/rsyslog restart  > /dev/null 2>&1
+
 	systemctl restart rsyslog.service > /dev/null 2>&1
+        systemctl restart syslog.service > /dev/null 2>&1
+        systemctl restart syslog-ng.service > /dev/null 2>&1
+
 fi
 
 }
@@ -185,26 +196,30 @@ if [ ! -z "$syslog_conf" ] ; then
 			egrep -qs 'MariaDB Columnstore Database Platform Logging' ${syslog_conf}
 			if [ $? -eq 0 ]; then
 				if [ -f ${syslog_conf}.columnstoreSave ] ; then
-					#uninstall the syslog for calpont logging
+					#uninstall the syslog for ColumnStore logging
-					mv -f ${syslog_conf} ${syslog_conf}.calpontBackup
+					sudo v -f ${syslog_conf} ${syslog_conf}.ColumnStoreBackup
-					mv -f ${syslog_conf}.columnstoreSave ${syslog_conf} >/dev/null 2>&1
+					sudo mv -f ${syslog_conf}.columnstoreSave ${syslog_conf} >/dev/null 2>&1
 					if [ ! -f ${syslog_conf} ] ; then
-						cp ${syslog_conf}.calpontBackup ${syslog_conf}
+						sudo cp ${syslog_conf}.ColumnStoreBackup ${syslog_conf}
 					fi
 				fi
 			fi
-			sed -i '/# MariaDB/,$d' ${syslog_conf} > /dev/null 2>&1
+			sudo sed -i '/# MariaDB/,$d' ${syslog_conf} > /dev/null 2>&1
 		else
-			rm -f "$syslog_conf"
+			sudo rm -f "$syslog_conf"
 		fi
 	else
-		rm -f "$syslog_conf"
+		sudo rm -f "$syslog_conf"
 	fi
 
-	pkill -hup syslogd > /dev/null 2>&1
+        sudo etc/init.d/rsyslog restart  > /dev/null 2>&1
-	pkill -hup syslog-ng  > /dev/null 2>&1
+        sudo /etc/init.d/syslog restart  > /dev/null 2>&1
-	/etc/init.d/rsyslog restart  > /dev/null 2>&1
+        sudo /etc/init.d/syslog-ng restart  > /dev/null 2>&1
-	systemctl restart rsyslog.service > /dev/null 2>&1
+
+        systemctl restart rsyslog.service > /dev/null 2>&1
+        systemctl restart syslog.service > /dev/null 2>&1
+        systemctl restart syslog-ng.service > /dev/null 2>&1
+
 
 	$installdir/bin/setConfig -d Installation SystemLogConfigFile "unassigned"
 
@@ -227,7 +242,7 @@ fi
 check() {
 test -f $installdir/post/functions && . $installdir/post/functions
 number=$RANDOM
-cplogger -i 104 "MariaDB Columnstore Log Test: $number"
+$installdir/bin/cplogger -i 104 "MariaDB Columnstore Log Test: $number"
 sleep 3
 egrep -qs "MariaDB Columnstore Log Test: $number" /var/log/mariadb/columnstore/info.log
 if [ $? -eq 0 ]; then

oam/oamcpp/liboamcpp.cpp

@@ -1367,13 +1367,13 @@ namespace oam
 				{
 					processor.shutdown();
 					string error = e.what();
-					writeLog("getSystemStatus: write exception: " + error, LOG_TYPE_ERROR);
+					//writeLog("getSystemStatus: write exception: " + error, LOG_TYPE_ERROR);
 					exceptionControl("getSystemStatus write", API_FAILURE);
 				}
 				catch(...)
 				{
 					processor.shutdown();
-					writeLog("getSystemStatus: write exception: unknown", LOG_TYPE_ERROR);
+					//writeLog("getSystemStatus: write exception: unknown", LOG_TYPE_ERROR);
 					exceptionControl("getSystemStatus write", API_FAILURE);
 				}
 
@@ -1386,13 +1386,13 @@ namespace oam
 				{
 					processor.shutdown();
 					string error = e.what();
-					writeLog("getSystemStatus: read exception: " + error, LOG_TYPE_ERROR);
+					//writeLog("getSystemStatus: read exception: " + error, LOG_TYPE_ERROR);
 					exceptionControl("getSystemStatus read", API_FAILURE);
 				}
 				catch(...)
 				{
 					processor.shutdown();
-					writeLog("getSystemStatus: read exception: unknown", LOG_TYPE_ERROR);
+					//writeLog("getSystemStatus: read exception: unknown", LOG_TYPE_ERROR);
 					exceptionControl("getSystemStatus read", API_FAILURE);
 				}
 
@@ -1475,21 +1475,21 @@ namespace oam
 				}
 				else
 				{
-					writeLog("getSystemStatus: ProcStatusControl returns 0 length", LOG_TYPE_ERROR);
+					//writeLog("getSystemStatus: ProcStatusControl returns 0 length", LOG_TYPE_ERROR);
 				}
 				// timeout ocurred, shutdown connection
 				processor.shutdown();
-				writeLog("getSystemStatus: read 0 length", LOG_TYPE_ERROR);
+				//writeLog("getSystemStatus: read 0 length", LOG_TYPE_ERROR);
 				exceptionControl("getSystemStatus read 0", API_FAILURE);
 			}
 			catch (exception& e)
 			{
 				string error = e.what();
-				writeLog("getSystemStatus: final exception: " + error, LOG_TYPE_ERROR);
+				//writeLog("getSystemStatus: final exception: " + error, LOG_TYPE_ERROR);
 			}
 			catch(...)
 			{
-				writeLog("getSystemStatus: final exception: unknown", LOG_TYPE_ERROR);
+				//writeLog("getSystemStatus: final exception: unknown", LOG_TYPE_ERROR);
 			}
 		}
 
@@ -1564,13 +1564,13 @@ namespace oam
 								Oam oam;
 								ostringstream os;
 								os << "Oam::getModuleStatus exception while getNICStatus " << (*pt1).HostName << " " << e.what();
-								oam.writeLog(os.str(), logging::LOG_TYPE_ERROR);
+								//oam.writeLog(os.str(), logging::LOG_TYPE_ERROR);
 							}
 							catch (...) {
 								Oam oam;
 								ostringstream os;
 								os << "Oam::getModuleStatus exception while getNICStatus " << (*pt1).HostName;
-								oam.writeLog(os.str(), logging::LOG_TYPE_ERROR);
+								//oam.writeLog(os.str(), logging::LOG_TYPE_ERROR);
 							}
 						}
 						
@@ -1589,13 +1589,13 @@ namespace oam
 						Oam oam;
 						ostringstream os;
 						os << "Oam::getModuleStatus exception while getSystemConfig " << name << " " << e.what();
-						oam.writeLog(os.str(), logging::LOG_TYPE_ERROR);
+						//oam.writeLog(os.str(), logging::LOG_TYPE_ERROR);
 					}
 					catch (...) {
 						Oam oam;
 						ostringstream os;
 						os << "Oam::getModuleStatus exception while getSystemConfig " << name;
-						oam.writeLog(os.str(), logging::LOG_TYPE_ERROR);
+						//oam.writeLog(os.str(), logging::LOG_TYPE_ERROR);
 					}
 				}
 			}
@@ -1605,13 +1605,13 @@ namespace oam
 			Oam oam;
 			ostringstream os;
 			os << "Oam::getModuleStatus exception while getSystemStatus " << e.what();
-			oam.writeLog(os.str(), logging::LOG_TYPE_ERROR);
+			//oam.writeLog(os.str(), logging::LOG_TYPE_ERROR);
 		}
 		catch (...) {
 			Oam oam;
 			ostringstream os;
 			os << "Oam::getModuleStatus exception while getSystemStatus";
-			oam.writeLog(os.str(), logging::LOG_TYPE_ERROR);
+			//oam.writeLog(os.str(), logging::LOG_TYPE_ERROR);
 		}
 
         // no match found
@@ -1795,7 +1795,7 @@ namespace oam
 			Oam oam;
 			ostringstream os;
 			os << "Oam::getNICStatus exception while getSystemStatus for " << name << " " << e.what();
-			oam.writeLog(os.str(), logging::LOG_TYPE_ERROR);
+			//oam.writeLog(os.str(), logging::LOG_TYPE_ERROR);
 	        exceptionControl("getNICStatus", API_FAILURE);
 		}
 
@@ -5984,7 +5984,15 @@ namespace oam
 
 		//format attached volume
 		writeLog("addUMdisk - Format new Volume for: " + volumeName, LOG_TYPE_DEBUG);
-		string cmd = "mkfs.ext2 -F " + device + " > /dev/null 2>&1";
+
+		string cmd;
+       	int user;
+        user = getuid();
+        if (user == 0)
+			cmd = "mkfs.ext2 -F " + device + " > /dev/null 2>&1";
+		else
+            cmd = "sudo mkfs.ext2 -F " + device + " > /dev/null 2>&1";
+
 		system(cmd.c_str());
 
 		//detach volume
@@ -6202,7 +6210,13 @@ namespace oam
 			
 				//format attached volume
 				cout << "  Formatting DBRoot #" << itoa(*pt1) << ", please wait..." << endl;
-				string cmd = "mkfs.ext2 -F " + amazonDeviceName + " > /tmp/format.log 2>&1";
+		        string cmd;
+        		int user;
+    		    user = getuid();
+     			if (user == 0)
+            		cmd = "mkfs.ext2 -F " + amazonDeviceName + " > /tmp/format.log 2>&1";
+        		else
+            		cmd = "sudo mkfs.ext2 -F " + amazonDeviceName + " > /tmp/format.log 2>&1";
 
 				writeLog("addDbroot format cmd: " + cmd, LOG_TYPE_DEBUG );
 
@@ -7567,7 +7581,7 @@ namespace oam
     std::string Oam::getEC2InstanceIpAddress(std::string instanceName)
 	{
 		// run script to get Instance status and IP Address
-		string cmd = InstallDir + "/bin/IDBInstanceCmds.sh getPrivateIP " + instanceName + " > /tmp/getCloudIP_" + instanceName;
+		string cmd = InstallDir + "/bin/MCSInstanceCmds.sh getPrivateIP " + instanceName + " > /tmp/getCloudIP_" + instanceName;
 		system(cmd.c_str());
 
 		if (checkLogStatus("/tmp/getCloudIP_" + instanceName, "stopped") )
@@ -7604,7 +7618,7 @@ namespace oam
     std::string Oam::getEC2LocalInstance(std::string name)
 	{
 		// run script to get Instance status and IP Address
-		string cmd = InstallDir + "/bin/IDBInstanceCmds.sh getInstance  > /tmp/getInstanceInfo_" + name;
+		string cmd = InstallDir + "/bin/MCSInstanceCmds.sh getInstance  > /tmp/getInstanceInfo_" + name;
 		int status = system(cmd.c_str());
 		if (WEXITSTATUS(status) != 0 )
 			return "failed";
@@ -7635,7 +7649,7 @@ namespace oam
     std::string Oam::getEC2LocalInstanceType(std::string name)
 	{
 		// run script to get Instance status and IP Address
-		string cmd = InstallDir + "/bin/IDBInstanceCmds.sh getType  > /tmp/getInstanceType_" + name;
+		string cmd = InstallDir + "/bin/MCSInstanceCmds.sh getType  > /tmp/getInstanceType_" + name;
 		int status = system(cmd.c_str());
 		if (WEXITSTATUS(status) != 0 )
 			return "failed";
@@ -7666,7 +7680,7 @@ namespace oam
     	std::string Oam::getEC2LocalInstanceSubnet(std::string name)
 	{
 		// run script to get Instance Subnet
-		string cmd = InstallDir + "/bin/IDBInstanceCmds.sh getSubnet  > /tmp/getInstanceSubnet_" + name;
+		string cmd = InstallDir + "/bin/MCSInstanceCmds.sh getSubnet  > /tmp/getInstanceSubnet_" + name;
 		int status = system(cmd.c_str());
 		if (WEXITSTATUS(status) != 0 )
 			return "failed";
@@ -7698,7 +7712,7 @@ namespace oam
     std::string Oam::launchEC2Instance( const std::string name, const std::string IPAddress, const std::string type, const std::string group)
 	{
 		// run script to get Instance status and IP Address
-		string cmd = InstallDir + "/bin/IDBInstanceCmds.sh launchInstance " + IPAddress + " " + type + " " + group + " > /tmp/getInstance_" + name;
+		string cmd = InstallDir + "/bin/MCSInstanceCmds.sh launchInstance " + IPAddress + " " + type + " " + group + " > /tmp/getInstance_" + name;
 		int status = system(cmd.c_str());
 		if (WEXITSTATUS(status) != 0 )
 			return "failed";
@@ -7740,7 +7754,7 @@ namespace oam
     void Oam::terminateEC2Instance(std::string instanceName)
 	{
 		// run script to get Instance status and IP Address
-		string cmd = InstallDir + "/bin/IDBInstanceCmds.sh terminateInstance " + instanceName + " > /tmp/terminateEC2Instance_" + instanceName;
+		string cmd = InstallDir + "/bin/MCSInstanceCmds.sh terminateInstance " + instanceName + " > /tmp/terminateEC2Instance_" + instanceName;
 		system(cmd.c_str());
 
 		return;
@@ -7757,7 +7771,7 @@ namespace oam
     void Oam::stopEC2Instance(std::string instanceName)
 	{
 		// run script to get Instance status and IP Address
-		string cmd = InstallDir + "/bin/IDBInstanceCmds.sh stopInstance " + instanceName + " > /tmp/stopEC2Instance_" + instanceName;
+		string cmd = InstallDir + "/bin/MCSInstanceCmds.sh stopInstance " + instanceName + " > /tmp/stopEC2Instance_" + instanceName;
 		system(cmd.c_str());
 
 		return;
@@ -7774,7 +7788,7 @@ namespace oam
     bool Oam::startEC2Instance(std::string instanceName)
 	{
 		// run script to get Instance status and IP Address
-		string cmd = InstallDir + "/bin/IDBInstanceCmds.sh startInstance " + instanceName + " > /tmp/startEC2Instance_" + instanceName;
+		string cmd = InstallDir + "/bin/MCSInstanceCmds.sh startInstance " + instanceName + " > /tmp/startEC2Instance_" + instanceName;
 		int ret = system(cmd.c_str());
 		if (WEXITSTATUS(ret) != 0 )
 			return false;
@@ -7793,7 +7807,7 @@ namespace oam
     bool Oam::assignElasticIP(std::string instanceName, std::string IpAddress)
 	{
 		// run script to get Instance status and IP Address
-		string cmd = InstallDir + "/bin/IDBInstanceCmds.sh assignElasticIP " + instanceName + " " + IpAddress + " > /tmp/assignElasticIP_" + instanceName;
+		string cmd = InstallDir + "/bin/MCSInstanceCmds.sh assignElasticIP " + instanceName + " " + IpAddress + " > /tmp/assignElasticIP_" + instanceName;
 		int ret = system(cmd.c_str());
 		if (WEXITSTATUS(ret) != 0 )
             exceptionControl("assignElasticIP", oam::API_FAILURE);
@@ -7812,7 +7826,7 @@ namespace oam
     bool Oam::deassignElasticIP(std::string IpAddress)
 	{
 		// run script to get Instance status and IP Address
-		string cmd = InstallDir + "/bin/IDBInstanceCmds.sh deassignElasticIP " + IpAddress + " > /tmp/deassignElasticIP_" + IpAddress;
+		string cmd = InstallDir + "/bin/MCSInstanceCmds.sh deassignElasticIP " + IpAddress + " > /tmp/deassignElasticIP_" + IpAddress;
 		int ret = system(cmd.c_str());
 		if (WEXITSTATUS(ret) != 0 )
             exceptionControl("deassignElasticIP", oam::API_FAILURE);
@@ -7831,7 +7845,7 @@ namespace oam
 	std::string Oam::getEC2VolumeStatus(std::string volumeName)
 	{
 		// run script to get Volume Status
-		string cmd = InstallDir + "/bin/IDBVolumeCmds.sh describe " + volumeName + " > /tmp/getVolumeStatus_" + volumeName;
+		string cmd = InstallDir + "/bin/MCSVolumeCmds.sh describe " + volumeName + " > /tmp/getVolumeStatus_" + volumeName;
 		int ret = system(cmd.c_str());
 		if (WEXITSTATUS(ret) != 0 )
 			return "failed";
@@ -7862,7 +7876,7 @@ namespace oam
 	std::string Oam::createEC2Volume(std::string size, std::string name)
 	{
 		// run script to get Volume Status
-		string cmd = InstallDir + "/bin/IDBVolumeCmds.sh create " + size + " " + name + " > /tmp/createVolumeStatus_" + name;
+		string cmd = InstallDir + "/bin/MCSVolumeCmds.sh create " + size + " " + name + " > /tmp/createVolumeStatus_" + name;
 		int ret = system(cmd.c_str());
 		if (WEXITSTATUS(ret) != 0 )
 			return "failed";
@@ -7906,7 +7920,7 @@ namespace oam
 		for ( int retry = 0 ; retry < 2 ; retry++ )
 		{
 			// run script to attach Volume
-			string cmd = InstallDir + "/bin/IDBVolumeCmds.sh attach " + volumeName + " " + instanceName + " " + deviceName + " > /tmp/attachVolumeStatus_" + volumeName;
+			string cmd = InstallDir + "/bin/MCSVolumeCmds.sh attach " + volumeName + " " + instanceName + " " + deviceName + " > /tmp/attachVolumeStatus_" + volumeName;
 			ret = system(cmd.c_str());
 
 			if (WEXITSTATUS(ret) == 0 )
@@ -7933,7 +7947,7 @@ namespace oam
 	bool Oam::detachEC2Volume(std::string volumeName)
 	{
 		// run script to attach Volume
-		string cmd = InstallDir + "/bin/IDBVolumeCmds.sh detach " + volumeName + " > /tmp/detachVolumeStatus_" + volumeName;
+		string cmd = InstallDir + "/bin/MCSVolumeCmds.sh detach " + volumeName + " > /tmp/detachVolumeStatus_" + volumeName;
 		int ret = system(cmd.c_str());
 		if (WEXITSTATUS(ret) != 0 )
 			return false;
@@ -7952,7 +7966,7 @@ namespace oam
 	bool Oam::deleteEC2Volume(std::string volumeName)
 	{
 		// run script to delete Volume
-		string cmd = InstallDir + "/bin/IDBVolumeCmds.sh delete " + volumeName + " > /tmp/deleteVolumeStatus_" + volumeName;
+		string cmd = InstallDir + "/bin/MCSVolumeCmds.sh delete " + volumeName + " > /tmp/deleteVolumeStatus_" + volumeName;
 		int ret = system(cmd.c_str());
 		if (WEXITSTATUS(ret) != 0 )
 			return false;
@@ -7971,7 +7985,7 @@ namespace oam
 	bool Oam::createEC2tag(std::string resourceName, std::string tagName, std::string tagValue)
 	{
 		// run script to create a tag
-		string cmd = InstallDir + "/bin/IDBVolumeCmds.sh createTag " + resourceName + " " + tagName + " " + tagValue + " > /tmp/createTagStatus_" + resourceName;
+		string cmd = InstallDir + "/bin/MCSVolumeCmds.sh createTag " + resourceName + " " + tagName + " " + tagValue + " > /tmp/createTagStatus_" + resourceName;
 		int ret = system(cmd.c_str());
 		if (WEXITSTATUS(ret) != 0 )
 			return false;
@@ -8540,17 +8554,32 @@ namespace oam
 	{
 		writeLog("updateFstab called: " + device + ":" + dbrootID, LOG_TYPE_DEBUG );
 
-		//check if entry already exist 
+		//check if entry already exist
-		string cmd = "grep /data" + dbrootID + " /etc/fstab > /dev/null 2>&1";
+        int user;
+        user = getuid();
+
+		string cmd;
+		if (user == 0)
+			cmd = "grep /data" + dbrootID + " /etc/fstab > /dev/null 2>&1";
+		else
+            cmd = "sudo grep /data" + dbrootID + " /etc/fstab > /dev/null 2>&1";
+
 		int status = system(cmd.c_str());
 		if (WEXITSTATUS(status) == 0 )
 			return "";
 
 		//update /etc/fstab with mount
-		string entry = device + " " + InstallDir + "/data" + dbrootID + " ext2 noatime,nodiratime,noauto 0 0";
+		string entry;
+		if (user == 0)
+			entry = device + " " + InstallDir + "/data" + dbrootID + " ext2 noatime,nodiratime,noauto 0 0";
+		else
+            entry = device + " " + InstallDir + "/data" + dbrootID + " ext2 noatime,nodiratime,noauto,user 0 0";
 
-		//update local fstab	
+		//update local fstab
-		cmd = "echo " + entry + " >> /etc/fstab";
+        if (user == 0)
+			cmd = "echo " + entry + " >> /etc/fstab";
+        else
+			cmd = "sudo echo " + entry + " >> /etc/fstab";
 		system(cmd.c_str());
 
 		//use from addmodule later
@@ -9614,7 +9643,7 @@ namespace oam
 			// not root user
 			// The stat above may fail for non-root because of permissions
 			// This is a non-optimal solution
-			string cmd = "pgrep ProcMon";
+			string cmd = "pgrep ProcMon > /dev/null 2>&1";
 			if (system(cmd.c_str()) == 0)
 			{
 				return true;

oamapps/alarmmanager/alarmmanager.cpp

@@ -234,7 +234,6 @@ void processAlarm(const Alarm& calAlarm)
            			rewriteActiveLog (alarmList);
 				} catch (runtime_error& e)
 				{
-					if (ALARM_DEBUG) {                                
 						LoggingID lid(11);                                
 						MessageLog ml(lid);                                
 						Message msg;                                
@@ -242,9 +241,7 @@ void processAlarm(const Alarm& calAlarm)
 						args.add("rewriteActiveLog error:");                                
 						args.add(e.what());                                
 						msg.format(args);                                
-						ml.logDebugMessage(msg);                                
+						ml.logErrorMessage(msg);                                
-					}
-					exit(1);
 				}
            		break;
            	}
@@ -256,7 +253,6 @@ void processAlarm(const Alarm& calAlarm)
    			logAlarm (calAlarm, ACTIVE_ALARM_FILE);
 		} catch (runtime_error& e)
 		{
-			if (ALARM_DEBUG) {                                
 				LoggingID lid(11);                                
 				MessageLog ml(lid);                                
 				Message msg;                                
@@ -264,9 +260,7 @@ void processAlarm(const Alarm& calAlarm)
 				args.add("logAlarm error:");                                
 				args.add(e.what());                                
 				msg.format(args);                                
-				ml.logDebugMessage(msg);                                
+				ml.logErrorMessage(msg);                                
-			}
-			exit(1);
 		}
 	}
 	
@@ -276,7 +270,6 @@ void processAlarm(const Alarm& calAlarm)
 			logAlarm (calAlarm, ALARM_FILE);
 		} catch (runtime_error& e)
 		{
-			if (ALARM_DEBUG) {                                
 				LoggingID lid(11);                                
 				MessageLog ml(lid);                                
 				Message msg;                                
@@ -284,9 +277,7 @@ void processAlarm(const Alarm& calAlarm)
 				args.add("logAlarm error:");                                
 				args.add(e.what());                                
 				msg.format(args);                                
-				ml.logDebugMessage(msg);                                
+				ml.logErrorMessage(msg);                                
-			}
-			exit(1);
 		}
 	}
 }
@@ -462,7 +453,6 @@ void ALARMManager::sendAlarmReport (const char* componentID, int alarmID, int st
   		configAlarm (calAlarm);
 	} catch (runtime_error& e)
 	{
-		if (ALARM_DEBUG) {                                
 			LoggingID lid(11);                                
 			MessageLog ml(lid);                                
 			Message msg;                                
@@ -470,9 +460,7 @@ void ALARMManager::sendAlarmReport (const char* componentID, int alarmID, int st
 			args.add("configAlarm error:");                                
 			args.add(e.what());                                
 			msg.format(args);                                
-			ml.logDebugMessage(msg);                                
+			ml.logErrorMessage(msg);                                
-		}
-		exit(1);
 	}
 
 	return;

oamapps/mcsadmin/CMakeLists.txt

@@ -8,7 +8,7 @@ set(mcsadmin_SRCS mcsadmin.cpp)
 
 add_executable(mcsadmin ${mcsadmin_SRCS})
 
-target_link_libraries(mcsadmin ${ENGINE_LDFLAGS} readline ncurses  ${ENGINE_EXEC_LIBS})
+target_link_libraries(mcsadmin ${ENGINE_LDFLAGS} readline ncurses  ${ENGINE_EXEC_LIBS} ${ENGINE_WRITE_LIBS})
 
 install(TARGETS mcsadmin DESTINATION ${ENGINE_BINDIR} COMPONENT platform)
 

oamapps/mcsadmin/mcsadmin.cpp

@@ -36,8 +36,8 @@ extern int h_errno;
 #include "messagequeue.h"
 #include "we_messages.h"
 #include "we_redistributedef.h"
-
+#include "we_config.h" // for findObjectFile
-
+#include "we_fileop.h" // for findObjectFile
 namespace fs = boost::filesystem;
 
 using namespace alarmmanager;
@@ -46,6 +46,7 @@ using namespace oam;
 using namespace config;
 using namespace messageqcpp;
 using namespace redistribute;
+using namespace execplan;
 
 #include "installdir.h"
 
@@ -860,8 +861,95 @@ int processCommand(string* arguments)
         }
         break;
 
-        case 5: // Available
+        case 5: // findObjectFile
         {
+			unsigned maxDBRoot = WriteEngine::Config::DBRootCount();
+			if (maxDBRoot < 1)
+			{
+				cout << endl << "getDatafileName fails because there are no dbroots defined for this server" << endl;
+				break;;
+			}
+			if (arguments[1] == "")
+			{
+				cout << endl << "getDatafileName requires one of" << endl;
+				cout << "a) oid of column for which file name is to be retrieved" << endl;
+				cout << "b) schema, table and column for which file name is to be retrieved" << endl;
+				break;
+			}
+			char* endchar;
+			int oid = strtol(arguments[1].c_str(), &endchar, 0);
+			// test to see if not all numeric
+			if (endchar < &(*arguments[1].end()))
+			{
+				oid = 0;
+			}
+			if (oid == 0)
+			{
+				// Need to convert the arguments to oid
+				boost::shared_ptr<execplan::CalpontSystemCatalog> systemCatalogPtr =
+					execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(0);
+				CalpontSystemCatalog::TableColName columnName;
+				columnName.schema = arguments[1];
+				if (arguments[2] == "")
+				{
+					cout << endl << "getDatafileName requires a table and column for schema " << arguments[1] << endl;
+					break;
+				}
+				columnName.table = arguments[2];
+				if (arguments[3] == "")
+				{
+					// No column was given. Use the first column in the table.
+					CalpontSystemCatalog::TableName tableName;
+					tableName.schema = arguments[1];
+					tableName.table = arguments[2];
+					CalpontSystemCatalog::RIDList rdlist = systemCatalogPtr->columnRIDs(tableName);
+					oid = rdlist.front().objnum;
+				}
+				else
+				{
+					columnName.column = arguments[3];
+					oid = systemCatalogPtr->lookupOID(columnName);
+				}
+			}
+
+			// Use writeengine code to get the filename
+			WriteEngine::FileOp fileOp;
+			char fileName[WriteEngine::FILE_NAME_SIZE];
+			memset(fileName, 0, WriteEngine::FILE_NAME_SIZE);
+			int rc;
+
+			if (oid < 1000)
+				rc = fileOp.getVBFileName(oid, fileName);
+			else
+				rc = fileOp.oid2DirName(oid, fileName);
+			cout << "file name for oid " << oid << ":" << endl;
+			if (strlen(fileName) > 0)
+			{
+				cout << fileName;
+			}
+			if (rc == WriteEngine::NO_ERROR)
+			{
+				// Success. No more output.
+				cout << endl;
+			}
+			else if (rc == WriteEngine::ERR_FILE_NOT_EXIST)
+			{
+				if (strlen(fileName) == 0)
+				{
+					// We couldn't get a name
+					cout << "Error: Filename could not be determined" << endl;
+				}
+				else
+				{
+					// We got a name, but the file doesn't exist
+					cout << " (OID directory not found)" << endl;
+				}
+			}
+			else
+			{
+				// Something broke
+				cerr << "WriteEngine::FileOp::oid2DirName() error. rc=" << rc << endl;
+			}
         }
         break;
 
@@ -4577,7 +4665,6 @@ int processCommand(string* arguments)
 			cout.width(20);
 			cout << "IP Address";
 			cout.width(14);
-			cout << "Status";
 			if ( AmazonElasticIPCount > 0 )
 			{
 				cout.width(20);
@@ -4656,16 +4743,6 @@ int processCommand(string* arguments)
 								cout << ipAddr;
 								cout.width(14);
 
-								try {
-									oam.getNICStatus(hostname, state);
-		
-									printState(state, " ");
-								}
-								catch (exception& e)
-								{
-									cout << INITIALSTATE;
-								}
-
 								if ( nicID == "1" && AmazonElasticIPCount > 0 )
 								{
 									int id = 1;
@@ -4736,65 +4813,6 @@ int processCommand(string* arguments)
 
 			cout << endl;
 
-			// get and all display Ext Devices Name config parameters
-
-			try
-			{
-	            SystemExtDeviceConfig systemextdeviceconfig;
-				oam.getSystemConfig(systemextdeviceconfig);
-		
-				if ( systemextdeviceconfig.Count == 0 )
-					break;
-
-				cout << endl << "External Device Configuration" << endl << endl;
-
-				cout.setf(ios::left);
-				cout.width(30);
-				cout << "Device Name";
-				cout.width(20);
-				cout << "IP Address";
-				cout.width(10);
-				cout << "Status";
-				cout << endl;
-				cout.width(30);
-				cout << "---------------------";
-				cout.width(20);
-				cout << "---------------";
-				cout.width(12);
-				cout << "------------";
-				cout << endl;
-
-				for ( unsigned int i = 0 ; i < systemextdeviceconfig.Count ; i++ )
-				{
-					cout.setf(ios::left);
-					cout.width(30);
-					cout << systemextdeviceconfig.extdeviceconfig[i].Name;
-					cout.width(20);
-					cout << systemextdeviceconfig.extdeviceconfig[i].IPAddr;
-					cout.width(12);
-
-					int state;
-					try {
-						oam.getExtDeviceStatus(systemextdeviceconfig.extdeviceconfig[i].Name, state);
-
-						printState(state, " ");
-					}
-					catch (exception& e)
-					{
-						cout << INITIALSTATE;
-					}
-					cout << endl;
-				}
-
-				cout << endl;
-			}
-			catch (exception& e)
-			{
-				cout << endl << "**** getextdeviceconfig Failed =  " << e.what() << endl;
-			}
-
-			cout << endl;
-
 			break;
 		}
 

oamapps/postConfigure/helpers.cpp

@@ -114,8 +114,10 @@ void dbrmDirCheck()
 		dbrmroot = sysConfig->getConfig(SystemSection, "DBRMRoot");
 		dbrmrootPrev = sysConfigPrev->getConfig(SystemSection, "DBRMRoot");
 	}
-	catch(...)
+	catch(const std::exception &exc)
-	{}
+	{
+	    std::cerr << exc.what() << std::endl;
+	}
 
 	if ( dbrmrootPrev.empty() )
 		return;
@@ -259,8 +261,10 @@ void mysqlSetup()
 	try {
 		oam.actionMysqlCalpont(MYSQL_START);
 	}
-	catch(...)
+	catch(const std::exception &exc)
-	{}
+	{
+	    std::cerr << exc.what() << std::endl;
+	}
 	sleep(2);
 	
 	for (;;)
@@ -289,7 +293,10 @@ void mysqlSetup()
 			try {
 				oam.setSystemConfig("MySQLPasswordConfig", "y");
 			}
-			catch(...) {}
+			catch(const std::exception &exc)
+			{
+			    std::cerr << exc.what() << std::endl;
+			}
 		}
 		else
 		{
@@ -303,8 +310,10 @@ void mysqlSetup()
 				try {
 					oam.actionMysqlCalpont(MYSQL_STOP);
 				}
-				catch(...)
+				catch(const std::exception &exc)
-				{}
+				{
+				    std::cerr << exc.what() << std::endl;
+				}
 				unlink("/tmp/idbmysql.log");
 				break;
 			}
@@ -344,8 +353,10 @@ int sendUpgradeRequest(int IserverTypeInstall, bool pmwithum)
 			if ( procstat.ProcessOpState == oam::ACTIVE)
 				break;
 		}
-		catch (exception& ex)
+		catch(const std::exception &exc)
-		{}
+		{
+		    std::cerr << exc.what() << std::endl;
+		}
 	}
 
 	SystemModuleTypeConfig systemmoduletypeconfig;
@@ -353,8 +364,10 @@ int sendUpgradeRequest(int IserverTypeInstall, bool pmwithum)
 	try{
 		oam.getSystemConfig(systemmoduletypeconfig);
 	}
-	catch (exception& ex)
+	catch(const std::exception &exc)
-	{}
+	{
+	    std::cerr << exc.what() << std::endl;
+	}
 
 	ByteStream msg;
 	ByteStream::byte requestID = RUNUPGRADE;
@@ -393,8 +406,10 @@ int sendUpgradeRequest(int IserverTypeInstall, bool pmwithum)
 						}
 					}
 				}
-				catch (exception& ex)
+				catch(const std::exception &exc)
-				{}
+				{
+				    std::cerr << exc.what() << std::endl;
+				}
 			}
 		}
 	}
@@ -417,8 +432,10 @@ int sendReplicationRequest(int IserverTypeInstall, std::string password, std::st
 	try{
 		oam.getSystemConfig(systemmoduletypeconfig);
 	}
-	catch (exception& ex)
+	catch(const std::exception &exc)
-	{}
+	{
+	    std::cerr << exc.what() << std::endl;
+	}
 
 	//get Primary (Master) UM
 	string masterModule = oam::UnassignedName;
@@ -474,7 +491,7 @@ int sendReplicationRequest(int IserverTypeInstall, std::string password, std::st
 							returnStatus = sendMsgProcMon( (*pt).DeviceName, msg1, requestID, 600 );
 			
 							if ( returnStatus != API_SUCCESS) {
-								cout << "ERROR: Error return in running the MariaDB Columnstore Master DB Distribute, check /tmp/master-dist.logs on " << masterModule << endl;
+								cout << endl << "ERROR: Error return in running the MariaDB Columnstore Master DB Distribute, check /tmp/master-dist*.logs on " << masterModule << endl;
 								return returnStatus;
 							}
 
@@ -486,7 +503,7 @@ int sendReplicationRequest(int IserverTypeInstall, std::string password, std::st
 							returnStatus = sendMsgProcMon( (*pt).DeviceName, msg, requestID, 30 );
 			
 							if ( returnStatus != API_SUCCESS) {
-								cout << "ERROR: Error return in running the MariaDB Columnstore Master replication, check /tmp/master-rep*.logs on " << masterModule << endl;
+								cout << endl << "ERROR: Error return in running the MariaDB Columnstore Master replication, check /tmp/master-rep*.logs on " << masterModule << endl;
 								return returnStatus;
 							}
 
@@ -517,7 +534,7 @@ int sendReplicationRequest(int IserverTypeInstall, std::string password, std::st
 							returnStatus = sendMsgProcMon( (*pt).DeviceName, msg, requestID, 30 );
 			
 							if ( returnStatus != API_SUCCESS) {
-								cout << "ERROR: Error return in running the MariaDB Columnstore Slave replication, check /tmp/slave-rep*.logs on " << (*pt).DeviceName << endl;
+								cout << endl << "ERROR: Error return in running the MariaDB Columnstore Slave replication, check /tmp/slave-rep*.logs on " << (*pt).DeviceName << endl;
 								return returnStatus;
 							}
 
@@ -526,12 +543,14 @@ int sendReplicationRequest(int IserverTypeInstall, std::string password, std::st
 					}
 					else
 					{
-						cout << "ERROR: Module not Active, replication not done on " << (*pt).DeviceName << endl;
+						cout << endl << "ERROR: Module not Active, replication not done on " << (*pt).DeviceName << endl;
 						pt++;
 					}
 				}
-				catch (exception& ex)
+				catch(const std::exception &exc)
-				{}
+				{
+				    std::cerr << exc.what() << std::endl;
+				}
 			}
 			else
 				pt++;
@@ -593,7 +612,9 @@ int sendMsgProcMon( std::string module, ByteStream msg, int requestID, int timeo
 				try {
 					receivedMSG = mqRequest.read(&ts);
 				}
-				catch (...) {
+				catch (const std::exception &exc)
+				{
+					std::cerr << exc.what() << std::endl;
 					return returnStatus;
 				}
 	
@@ -630,8 +651,10 @@ int sendMsgProcMon( std::string module, ByteStream msg, int requestID, int timeo
 
 		mqRequest.shutdown();
 	}
-	catch (exception& ex)
+	catch(const std::exception &exc)
-	{}
+	{
+	    std::cerr << exc.what() << std::endl;
+	}
 
 	return returnStatus;
 }
@@ -651,8 +674,10 @@ void checkFilesPerPartion(int DBRootCount, Config* sysConfig)
 	try {
 		dbRoot = sysConfig->getConfig(SystemSection, "DBRoot1");
 	}
-	catch(...)
+	catch(const std::exception &exc)
-	{}
+	{
+	    std::cerr << exc.what() << std::endl;
+	}
 
 	dbRoot = dbRoot + "/000.dir";
 
@@ -661,8 +686,10 @@ void checkFilesPerPartion(int DBRootCount, Config* sysConfig)
 		string tmp = sysConfig->getConfig("ExtentMap", "FilesPerColumnPartition");
 		FilesPerColumnPartition = atoi(tmp.c_str());
 	}
-	catch(...)
+	catch(const std::exception &exc)
-	{}
+	{
+	    std::cerr << exc.what() << std::endl;
+	}
 
 	if ( fmod(FilesPerColumnPartition , (float) DBRootCount) != 0 ) {
 		ifstream oldFile (dbRoot.c_str());
@@ -738,8 +765,10 @@ void checkMysqlPort( std::string& mysqlPort, Config* sysConfig )
 				try {
 					sysConfig->setConfig("Installation", "MySQLPort", mysqlPort);
 				}
-				catch(...)
+				catch(const std::exception &exc)
-				{}
+				{
+				    std::cerr << exc.what() << std::endl;
+				}
 		
 				if ( !writeConfig(sysConfig) ) {
 					cout << "ERROR: Failed trying to update MariDB Columnstore System Configuration file" << endl;
@@ -860,8 +889,10 @@ void checkSystemMySQLPort(std::string& mysqlPort, Config* sysConfig, std::string
 			try {
 				sysConfig->setConfig("Installation", "MySQLPort", mysqlPort);
 			}
-			catch(...)
+			catch(const std::exception &exc)
-			{}
+			{
+			    std::cerr << exc.what() << std::endl;
+			}
 	
 			if ( !writeConfig(sysConfig) ) {
 				cout << "ERROR: Failed trying to update MariDB Columnstore System Configuration file" << endl;
@@ -889,8 +920,10 @@ bool writeConfig( Config* sysConfig )
 			sysConfig->write();
 			return true;
 		}
-		catch(...)
+		catch(const std::exception &exc)
-		{}
+		{
+		    std::cerr << exc.what() << std::endl;
+		}
 	}
 
 	return false;

oamapps/postConfigure/installer.cpp

@@ -289,7 +289,7 @@ int main(int argc, char *argv[])
 							value = "8G";
 					}
 
-					cout << "      Setting 'TotalUmMemory' to " << percent << "% of total memory (Combined Server Install maximum value is 16G). Value set to " << value << endl;
+					cout << "      Setting 'TotalUmMemory' to " << percent << "% of total memory. Value set to " << value << endl;
 	
 					try {
 						sysConfig->setConfig("HashJoin", "TotalUmMemory", value);

oamapps/postConfigure/postConfigure.cpp

@@ -209,6 +209,7 @@ int main(int argc, char *argv[])
 	noPrompting = false;
 	string password;
 	string cmd;
+	bool disableAmazon = false;
 //  	struct sysinfo myinfo; 
 
 	// hidden options
@@ -276,15 +277,17 @@ int main(int argc, char *argv[])
 			cout << "	Enter one of the options within [], if available, or" << endl;
 			cout << "	Enter a new value" << endl << endl;
 			cout << endl;
-   			cout << "Usage: postConfigure [-h][-c][-u][-p][-mp][-s][-port]" << endl;
+   			cout << "Usage: postConfigure [-h][-c][-u][-p][-mp][-s][-port][-i][-da]" << endl;
 			cout << "   -h  Help" << endl;
 			cout << "   -c  Config File to use to extract configuration data, default is Columnstore.xml.rpmsave" << endl;
 			cout << "   -u  Upgrade, Install using the Config File from -c, default to Columnstore.xml.rpmsave" << endl;
-			cout << "	 If ssh-keys aren't setup, you should provide passwords as command line arguments" << endl;
+			cout << "	    If ssh-keys aren't setup, you should provide passwords as command line arguments" << endl;
 			cout << "   -p  Unix Password, used with no-prompting option" << endl;
 			cout << "   -mp MariaDB Columnstore Password" << endl;
 			cout << "   -s  Single Threaded Remote Install" << endl;
 			cout << "   -port MariaDB Columnstore Port Address" << endl;
+            cout << "   -i Non-root Install directory, Only use for non-root installs" << endl;
+            cout << "   -da Disable Amazon functionality, install using Stardard Hostnames and IP Addresses" << endl;
 			exit (0);
 		}
       		else if( string("-s") == argv[i] )
@@ -336,18 +339,6 @@ int main(int argc, char *argv[])
 		// for backward compatibility
 		else if( string("-n") == argv[i] )
 			noPrompting = true;
-		else if( string("-i") == argv[i] ) {
-			i++;
-			if (i >= argc ) {
-				cout << "   ERROR: install dir not provided" << endl;
-				exit (1);
-			}
-			installDir = argv[i];
-			if ( installDir.find("-") != string::npos ) {
-				cout << "   ERROR: Valid install dir not provided" << endl;
-				exit (1);
-			}			
-		}
 		else if( string("-port") == argv[i] ) {
 			i++;
 			if (i >= argc ) {
@@ -361,10 +352,20 @@ int main(int argc, char *argv[])
 				exit (1);
 			}
 		}
+        else if( string("-da") == argv[i] )
+            disableAmazon = true;
+        else if( string("-i") == argv[i] ) {
+            i++;
+            if (i >= argc ) {
+                cout << "   ERROR: Path not provided" << endl;
+                exit (1);
+            }
+            installDir = argv[i];
+        }
 		else
 		{
 			cout << "   ERROR: Invalid Argument = " << argv[i] << endl;
-   			cout << "   Usage: postConfigure [-h][-c][-u][-p][-mp][-s][-port]" << endl;
+   			cout << "   Usage: postConfigure [-h][-c][-u][-p][-mp][-s][-port][-i][-da]" << endl;
 			exit (1);
 		}
 	}
@@ -814,31 +815,28 @@ int main(int argc, char *argv[])
 
 	//amazon install setup check
 	bool amazonInstall = false;
-	if (rootUser)
+	string amazonSubNet = oam::UnassignedName;
+	string cloud = oam::UnassignedName;
+	if (!disableAmazon)
+	{
 		system("ec2-version > /tmp/amazon.log 2>&1");
-	else
-		system("sudo ec2-version > /tmp/amazon.log 2>&1");
 
-	ifstream in("/tmp/amazon.log");
+		ifstream in("/tmp/amazon.log");
 
-	in.seekg(0, std::ios::end);
+		in.seekg(0, std::ios::end);
-	int size = in.tellg();
+		int size = in.tellg();
-	if ( size == 0 || oam.checkLogStatus("/tmp/amazon.log", "not found")) 
+		if ( size == 0 || oam.checkLogStatus("/tmp/amazon.log", "not found")) 
-	// not running on amazon with ec2-api-tools
+		// not running on amazon with ec2-api-tools
-		amazonInstall = false;
-	else
-		if ( size == 0 || oam.checkLogStatus("/tmp/amazon.log", "not installed")) 
 			amazonInstall = false;
 		else
-			amazonInstall = true;
+			if ( size == 0 || oam.checkLogStatus("/tmp/amazon.log", "not installed")) 
-
+				amazonInstall = false;
-	string amazonSubNet = oam::UnassignedName;
+			else
+				amazonInstall = true;
+	}
 
 	if ( amazonInstall )
 	{
-		string cloud = oam::UnassignedName;
-		string option = "1";
-
 		try {
 			cloud = sysConfig->getConfig(InstallSection, "Cloud");
 		}
@@ -847,69 +845,30 @@ int main(int argc, char *argv[])
 			cloud  = oam::UnassignedName;
 		}
 
-		if ( cloud  == oam::UnassignedName )
+		cout << endl << "Amazon EC2 Install, these files will need to be installed on the local instance:" << endl << endl;
-		    option = "2";
+		cout << " 1. File containing the Amazon Access Key" << endl;
-		  
+		cout << " 2. File containing the Amazon Secret Key" << endl << endl;
-		cout << "===== Amazon EC2-API-TOOLS Instance Install =====" << endl << endl;
-		cout << "You have 2 install options: " << endl << endl;
-		cout << "1. Utilizing the Amazon IDs for instances and volumes which allows for features like" << endl;
-		cout <<     "automaticly launching instances and EBS volumes when configuring and system expansion." << endl;
-		cout <<     "This option is recommended and would be use if you are setting up a MariaDB Columnstore system." << endl << endl;
-		cout << "2. Using standard hardware IDs for hostnames, IP Addresses, and Storage Devices." << endl;
-		cout <<     "Using this option, you would need to pre-create the Instances and the EBS storages" << endl;
-		cout <<     "and then provide the hostnames/IP-Addresses during the configuration and system expansion" << endl;
-		cout <<     "commands. This option would be used when you are installing on a existing system." << endl << endl;
 
 		while(true) {
-			prompt = "Select Install Option [1,2] (" + option + ") > ";
+			string ready = "y";
+			prompt = "Are these files installed and ready to continue [y,n] (y) > ";
 			pcommand = callReadline(prompt.c_str());
-			if (pcommand) {
+			if (pcommand) {	
-				if (strlen(pcommand) > 0) option = pcommand;
+				if (strlen(pcommand) > 0) ready = pcommand;
 				callFree(pcommand);
-			}
+				if (ready == "n") {
-	
+					cout << endl << "Please Install these files and re-run postConfigure. exiting..." << endl;
-			if (option == "2")
+					exit(0);
-			{
+				}
-				amazonInstall = false;
+
-				break;
+				if ( ready != "y" )
-			}
-			else
-			{
-				if ( option != "1" )
 				{
-					cout << "Invalid Entry, please enter '1' or '2'" << endl;
+					cout << "Invalid Entry, please enter 'y' for yes or 'n' for no" << endl;
 					if ( noPrompting )
 						exit(1);
-					continue;
 				}
 			}
 
-			cout << endl << "To use the EC2-api-tools, these files will need to be installed on" << endl;
-			cout << "on the local instance:" << endl << endl;
-			cout << " 1. File containing the Amazon Access Key" << endl;
-			cout << " 2. File containing the Amazon Secret Key" << endl << endl;
-
-			while(true) {
-				string ready = "y";
-				prompt = "Are these files installed and ready to continue [y,n] (y) > ";
-				pcommand = callReadline(prompt.c_str());
-				if (pcommand) {
-					if (strlen(pcommand) > 0) ready = pcommand;
-					callFree(pcommand);
-					if (ready == "n") {
-						cout << endl << "Please Install these files and re-run postConfigure. exiting..." << endl;
-						exit(0);
-					}
-				}
-
-				if ( ready == "y" )
-					break;
-
-				cout << "Invalid Entry, please enter 'y' for yes or 'n' for no" << endl;
-				if ( noPrompting )
-					exit(1);
-			}
-
 			try {
 				AmazonAccessKey = sysConfig->getConfig(InstallSection, "AmazonAccessKey");
 				AmazonSecretKey = sysConfig->getConfig(InstallSection, "AmazonSecretKey");
@@ -966,30 +925,30 @@ int main(int argc, char *argv[])
 			if( !copyKeyfiles() )
 				cout << "copyKeyfiles error" << endl;
 
-			try {
-				AmazonRegion = sysConfig->getConfig(InstallSection, "AmazonRegion");
-			}
-			catch(...)
-			{}
-
-			cout << endl;
-
-			prompt = "Enter Amazon Region you are running in (" + AmazonRegion + ") > ";
-			pcommand = callReadline(prompt.c_str());
-			if (pcommand) {
-				if (strlen(pcommand) > 0) AmazonRegion = pcommand;
-				callFree(pcommand);
-			}
-	
-			try {
-				sysConfig->setConfig(InstallSection, "AmazonRegion", AmazonRegion);
-			}
-			catch(...)
-			{}
-
 			break;
 		}
 
+		try {
+			AmazonRegion = sysConfig->getConfig(InstallSection, "AmazonRegion");
+		}
+		catch(...)
+		{}
+
+		cout << endl;
+
+		prompt = "Enter Amazon Region you are running in (" + AmazonRegion + ") > ";
+		pcommand = callReadline(prompt.c_str());
+		if (pcommand) {
+			if (strlen(pcommand) > 0) AmazonRegion = pcommand;
+			callFree(pcommand);
+		}
+	
+		try {
+			sysConfig->setConfig(InstallSection, "AmazonRegion", AmazonRegion);
+		}
+		catch(...)
+		{}
+
 		if ( !writeConfig(sysConfig) ) { 
 			cout << "ERROR: Failed trying to update MariaDB Columnstore System Configuration file" << endl; 
 			exit(1);
@@ -997,54 +956,51 @@ int main(int argc, char *argv[])
 
 		sleep(1);
 
-		if ( amazonInstall )
+		//get subnetID
-		{
+	    try {
-		      //get subnetID
+		      amazonSubNet = sysConfig->getConfig(InstallSection, "AmazonSubNetID");
-		      try {
+		}
-			      amazonSubNet = sysConfig->getConfig(InstallSection, "AmazonSubNetID");
+		catch(...)
-		      }
+      	{}
-		      catch(...)
-		      {}
 
-		      if ( amazonSubNet == oam::UnassignedName )
+		if ( amazonSubNet == oam::UnassignedName )
-		      {	
+		{	
-			      //check if this is a vpc system by checking for subnet setup
+			//check if this is a vpc system by checking for subnet setup
-			      amazonSubNet = oam.getEC2LocalInstanceSubnet();
+			amazonSubNet = oam.getEC2LocalInstanceSubnet();
-			      // cout << "amazonSubNet = " <<  amazonSubNet << endl;
+			// cout << "amazonSubNet = " <<  amazonSubNet << endl;
-			      if ( amazonSubNet == "failed" || amazonSubNet == "" )
+			if ( amazonSubNet == "failed" || amazonSubNet == "" )
-			      {
+			{
-				      amazonSubNet = oam::UnassignedName;
+				amazonSubNet = oam::UnassignedName;
-				      cloud = "amazon-ec2";
+				cloud = "amazon-ec2";
-			      }
+			}
-			      else
+			else
-			      {
+			{
-				      cloud = "amazon-vpc";
+				cloud = "amazon-vpc";
-			      }
+			}
 
-			      //set subnetID
+			//set subnetID
-			      try {
+			try {
-				      sysConfig->setConfig(InstallSection, "AmazonSubNetID", amazonSubNet);
+				sysConfig->setConfig(InstallSection, "AmazonSubNetID", amazonSubNet);
-			      }
+			}
-			      catch(...)
+			catch(...)
-			      {}
+			{}
-		      }
-		      else
-			      cloud = "amazon-vpc";
-
-		      try {
-			      sysConfig->setConfig(InstallSection, "Cloud", cloud);
-		      }
-		      catch(...)
-		      {}
 		}
 		else
-		{
+			cloud = "amazon-vpc";
-		    try {
+
-			      sysConfig->setConfig(InstallSection, "Cloud", oam::UnassignedName);
+		try {
-		    }
+			sysConfig->setConfig(InstallSection, "Cloud", cloud);
-		    catch(...)
-		    {}
 		}
+		catch(...)
+		{}
+	}
+	else
+	{
+		try {
+			sysConfig->setConfig(InstallSection, "Cloud", oam::UnassignedName);
+		}
+		catch(...)
+		{}
 	}
 
 	if ( pmwithum )
@@ -1083,7 +1039,7 @@ int main(int argc, char *argv[])
 	catch (exception& e) {}
 
 	//get Parent OAM Module Name
-	string parentOAMModuleName = "pm1";
+	parentOAMModuleName = "pm1";
 
 	if ( localModuleName != parentOAMModuleName )
 	{
@@ -2754,6 +2710,7 @@ int main(int argc, char *argv[])
 	/* create a thread_data_t argument array */
 	thread_data_t thr_data[childmodulelist.size()];
 
+	string install = "y";
 	if ( IserverTypeInstall != oam::INSTALL_COMBINE_DM_UM_PM || 
 			pmNumber > 1 ) {
 		//
@@ -2761,7 +2718,6 @@ int main(int argc, char *argv[])
 		//
 		cout << endl << "===== System Installation =====" << endl << endl;
 	
-		string install = "y";
 		cout << "System Configuration is complete, System Installation is the next step." << endl;
 
 		while(true)
@@ -2826,13 +2782,22 @@ int main(int argc, char *argv[])
 			}
 
 			if ( EEPackageType == "rpm" )
-				cout << "Performing an MariaDB Columnstore System install using RPM packages located in the " + HOME + " directory." << endl;
+			{
+				cout << "Performing an MariaDB Columnstore System install using RPM packages" << endl; 
+				cout << " located in the " + HOME + " directory." << endl;
+			}
 			else
 			{
 				if ( EEPackageType == "binary" )
-					cout << "Performing an MariaDB Columnstore System install using a Binary package located in the " + HOME + " directory." << endl;
+				{
+					cout << "Performing an MariaDB Columnstore System install using a Binary package" << endl; 
+					cout << "located in the " + HOME + " directory." << endl;
+				}
 				else
-					cout << "Performing an MariaDB Columnstore System install using using DEB packages located in the " + HOME + " directory." << endl;
+				{
+					cout << "Performing an MariaDB Columnstore System install using using DEB packages" << endl;
+					cout << "located in the " + HOME + " directory." << endl;
+				}
 			}
 	
 			//Write out Updated System Configuration File
@@ -2936,8 +2901,9 @@ int main(int argc, char *argv[])
 					break;
 				}
 
-				if ( strcmp(pass1, "exit") == 0 )
+				if ( pass1 == "exit")
 					exit(0);
+
 				string p1 = pass1;
 				pass2=getpass("Confirm password > ");
 				string p2 = pass2;
@@ -3397,16 +3363,19 @@ int main(int argc, char *argv[])
 					string remoteModuleName = (*list1).moduleName;
 					string remoteModuleIP = (*list1).moduleIP;
 					string remoteHostName = (*list1).hostName;
-			
+		
 					//run remote command script
 					cout << endl << "----- Starting MariaDB Columnstore on '" + remoteModuleName + "' -----" << endl << endl;
 
-					cmd = installDir + "/bin/remote_scp_put.sh " + remoteModuleIP + " " + installDir + "/etc/Columnstore.xml " +  installDir + "/etc/. > /dev/null 2>&1";
+					if ( install == "n" ) 
+					{	// didnt do a full install, push the config file
+						cmd = installDir + "/bin/remote_scp_put.sh " + remoteModuleIP + " " + installDir + "/etc/Columnstore.xml  > /dev/null 2>&1";
+						system(cmd.c_str());
+					}
+
+					cmd = installDir + "/bin/remote_command.sh " + remoteModuleIP + " " + password + " '" + installDir + "/bin/columnstore restart' 0";
 					int rtnCode = system(cmd.c_str());
 
-					cmd = installDir + "/bin/remote_command.sh " + remoteModuleIP + " " + password +
-						" '" + installDir + "/bin/columnstore restart' 0";
-					rtnCode = system(cmd.c_str());
 					if (WEXITSTATUS(rtnCode) != 0)
 						cout << "Error with running remote_command.sh" << endl;
 					else
@@ -4990,7 +4959,12 @@ void setSystemName()
  */
 bool copyFstab(string moduleName)
 {
-	string cmd = "/bin/cp -f /etc/fstab " + installDir + "/local/etc/" + moduleName + "/. > /dev/null 2>&1";
+	string cmd;	
+	if ( rootUser)
+   		cmd = "/bin/cp -f /etc/fstab " + installDir + "/local/etc/" + moduleName + "/. > /dev/null 2>&1";
+	else
+		cmd = "/sudo bin/cp -f /etc/fstab " + installDir + "/local/etc/" + moduleName + "/. > /dev/null 2>&1";
+
 	system(cmd.c_str());
 
 	return true;
@@ -5042,15 +5016,6 @@ bool updateBash()
 
    	ifstream newFile (fileName.c_str());
 
-	if (!rootUser)
-	{
-		string cmd = "echo export columnstore_INSTALL_DIR=" + installDir + " >> " + fileName;
-		system(cmd.c_str());
-	
-		cmd = "echo export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$columnstore_INSTALL_DIR/lib:$columnstore_INSTALL_DIR/mysql/lib/mysql >> " + fileName;
-		system(cmd.c_str());
-	}
-
 	if ( hdfs ) 
 	{	
 		string cmd = "echo . " + installDir + "/bin/" + DataFileEnvFile + " >> " + fileName;

procmgr/processmanager.cpp

@@ -4471,7 +4471,7 @@ int ProcessManager::addModule(oam::DeviceNetworkList devicenetworklist, std::str
 		if ( packageType == "deb" )
 			separator = "_";
 		//mariadb
-		calpontPackage = homedir + "/mariadb-columnstore-*" + separator + systemsoftware.Version + "-" + systemsoftware.Release + "*." + packageType;
+		calpontPackage = homedir + "/mariadb-columnstore*" + separator + systemsoftware.Version + "-" + systemsoftware.Release + "*." + packageType;
 		mysqlPackage = homedir + "/mariadb-columnstore-storage-engine" + separator + systemsoftware.Version + "-" + systemsoftware.Release + "*." + packageType;
 		mysqldPackage = homedir + "/mariadb-columnstore-mysql" + separator + systemsoftware.Version + "-" + systemsoftware.Release + "*." + packageType;
 		calpontPackage1 = homedir + "/mariadb-columnstore-libs" + separator + systemsoftware.Version + "-" + systemsoftware.Release + "*." + packageType;

utils/idbdatafile/IDBFileSystem.h

@@ -84,6 +84,13 @@ public:
 	 */
 	virtual off64_t size(const char* path) const = 0;
 
+    /**
+     * compressedSize() returns the decompressed size of the file
+     * speicified by path.
+     * Returns the size on success, -1 on error
+     */
+    virtual off64_t compressedSize(const char* path) const = 0;
+
 	/**
 	 * exists() checks for the existence of a particular path.
 	 * Returns true if exists, false otherwise.

utils/idbdatafile/IDBPolicy.h

@@ -115,6 +115,7 @@ public:
 	 */
 	static int mkdir(const char *pathname);
 	static off64_t size(const char* path);
+    static off64_t compressedSize(const char* path);
 	static int remove(const char *pathname);
 	static int rename(const char *oldpath, const char *newpath);
 	static bool exists(const char* pathname);
@@ -175,6 +176,12 @@ off64_t IDBPolicy::size(const char* path)
 	return IDBPolicy::getFs( path ).size( path );
 }
 
+inline
+off64_t IDBPolicy::compressedSize(const char* path)
+{
+    return IDBPolicy::getFs( path ).compressedSize( path );
+}
+
 inline
 int IDBPolicy::remove(const char *pathname)
 {

utils/idbdatafile/PosixFileSystem.cpp

@@ -17,6 +17,7 @@
 
 #include "PosixFileSystem.h"
 #include "IDBLogger.h"
+#include "idbcompress.h"
 
 #include <sys/stat.h>
 #include <unistd.h>
@@ -128,6 +129,103 @@ off64_t PosixFileSystem::size(const char* path) const
 	return ret;
 }
 
+size_t readFillBuffer(
+	idbdatafile::IDBDataFile* pFile,
+	char*   buffer,
+	size_t  bytesReq)
+{
+	char*   pBuf = buffer;
+	ssize_t nBytes;
+	size_t  bytesToRead = bytesReq;
+	size_t  totalBytesRead = 0;
+
+	while (1)
+	{
+		nBytes = pFile->read(pBuf, bytesToRead);
+		if (nBytes > 0)
+			totalBytesRead += nBytes;
+		else
+			break;
+
+		if ((size_t)nBytes == bytesToRead)
+			break;
+
+		pBuf        += nBytes;
+		bytesToRead  =  bytesToRead - (size_t)nBytes;
+	}
+
+	return totalBytesRead;
+}
+
+off64_t PosixFileSystem::compressedSize(const char *path) const
+{
+    IDBDataFile *pFile = NULL;
+    size_t nBytes;
+    off64_t dataSize = 0;
+
+    try
+    {
+        pFile = IDBDataFile::open(IDBDataFile::BUFFERED, path, "r", 0);
+
+        if (!pFile)
+        {
+            return -1;
+        }
+
+        compress::IDBCompressInterface decompressor;
+
+        char hdr1[compress::IDBCompressInterface::HDR_BUF_LEN];
+        nBytes = readFillBuffer( pFile,hdr1,compress::IDBCompressInterface::HDR_BUF_LEN);
+        if ( nBytes != compress::IDBCompressInterface::HDR_BUF_LEN )
+        {
+            delete pFile;
+            return -1;
+        }
+
+        // Verify we are a compressed file
+        if (decompressor.verifyHdr(hdr1) < 0)
+        {
+            delete pFile;
+            return -1;
+        }
+
+        int64_t ptrSecSize = decompressor.getHdrSize(hdr1) - compress::IDBCompressInterface::HDR_BUF_LEN;
+        char* hdr2 = new char[ptrSecSize];
+        nBytes = readFillBuffer( pFile,hdr2,ptrSecSize);
+        if ( (int64_t)nBytes != ptrSecSize )
+        {
+            delete[] hdr2;
+            delete pFile;
+            return -1;
+        }
+
+        compress::CompChunkPtrList chunkPtrs;
+        int rc = decompressor.getPtrList(hdr2, ptrSecSize, chunkPtrs);
+        delete[] hdr2;
+        if (rc != 0)
+        {
+            delete pFile;
+            return -1;
+        }
+
+        unsigned k = chunkPtrs.size();
+        // last header's offset + length will be the data bytes
+        if (k < 1)
+        {
+            delete pFile;
+            return -1;
+        }
+        dataSize = chunkPtrs[k-1].first + chunkPtrs[k-1].second;
+        delete pFile;
+        return dataSize;
+    }
+    catch (...)
+    {
+        delete pFile;
+        return -1;
+    }
+}
+
 bool PosixFileSystem::exists(const char *pathname) const
 {
 	boost::filesystem::path dirPath(pathname);

utils/idbdatafile/PosixFileSystem.h

@@ -31,6 +31,7 @@ public:
 
 	/* virtual */ int mkdir(const char *pathname);
 	/* virtual */ off64_t size(const char* path) const;
+    /* virtual */ off64_t compressedSize(const char *path) const;
 	/* virtual */ int remove(const char *pathname);
 	/* virtual */ int rename(const char *oldpath, const char *newpath);
 	/* virtual */ bool exists(const char* pathname) const;

utils/idbhdfs/hdfs-shared/HdfsFileSystem.cpp

@@ -31,6 +31,7 @@
 
 #include "HdfsFsCache.h"
 #include "IDBLogger.h"
+#include "idbcompress.h"
 
 #include <iostream>
 #include <list>
@@ -119,6 +120,104 @@ off64_t HdfsFileSystem::size(const char* path) const
 	return retval;
 }
 
+size_t readFillBuffer(
+	idbdatafile::IDBDataFile* pFile,
+	char*   buffer,
+	size_t  bytesReq)
+{
+	char*   pBuf = buffer;
+	ssize_t nBytes;
+	size_t  bytesToRead = bytesReq;
+	size_t  totalBytesRead = 0;
+
+	while (1)
+	{
+		nBytes = pFile->read(pBuf, bytesToRead);
+		if (nBytes > 0)
+			totalBytesRead += nBytes;
+		else
+			break;
+
+		if ((size_t)nBytes == bytesToRead)
+			break;
+
+		pBuf        += nBytes;
+		bytesToRead  =  bytesToRead - (size_t)nBytes;
+	}
+
+	return totalBytesRead;
+}
+
+
+off64_t HdfsFileSystem::compressedSize(const char *path) const
+{
+    IDBDataFile *pFile = NULL;
+    size_t nBytes;
+    off64_t dataSize = 0;
+
+    try
+    {
+        pFile = IDBDataFile::open(IDBDataFile::HDFS, path, "r", 0);
+
+        if (!pFile)
+        {
+            return -1;
+        }
+
+        compress::IDBCompressInterface decompressor;
+
+        char hdr1[compress::IDBCompressInterface::HDR_BUF_LEN];
+        nBytes = readFillBuffer( pFile,hdr1,compress::IDBCompressInterface::HDR_BUF_LEN);
+        if ( nBytes != compress::IDBCompressInterface::HDR_BUF_LEN )
+        {
+            delete pFile;
+            return -1;
+        }
+
+        // Verify we are a compressed file
+        if (decompressor.verifyHdr(hdr1) < 0)
+        {
+            delete pFile;
+            return -1;
+        }
+
+        int64_t ptrSecSize = decompressor.getHdrSize(hdr1) - compress::IDBCompressInterface::HDR_BUF_LEN;
+        char* hdr2 = new char[ptrSecSize];
+        nBytes = readFillBuffer( pFile,hdr2,ptrSecSize);
+        if ( (int64_t)nBytes != ptrSecSize )
+        {
+            delete[] hdr2;
+            delete pFile;
+            return -1;
+        }
+
+        compress::CompChunkPtrList chunkPtrs;
+        int rc = decompressor.getPtrList(hdr2, ptrSecSize, chunkPtrs);
+        delete[] hdr2;
+        if (rc != 0)
+        {
+            delete pFile;
+            return -1;
+        }
+
+        unsigned k = chunkPtrs.size();
+        // last header's offset + length will be the data bytes
+        if (k < 1)
+        {
+            delete pFile;
+            return -1;
+        }
+        dataSize = chunkPtrs[k-1].first + chunkPtrs[k-1].second;
+        delete pFile;
+        return dataSize;
+    }
+    catch (...)
+    {
+        delete pFile;
+        return -1;
+    }
+}
+
 bool HdfsFileSystem::exists(const char *pathname) const
 {
 	int ret = hdfsExists(m_fs,pathname);

utils/idbhdfs/hdfs-shared/HdfsFileSystem.h

@@ -45,6 +45,7 @@ public:
 
 	/* virtual */ int mkdir(const char *pathname);
 	/* virtual */ off64_t size(const char* path) const;
+    /* virtual */ off64_t compressedSize(const char* path) const;
 	/* virtual */ int remove(const char *pathname);
 	/* virtual */ int rename(const char *oldpath, const char *newpath);
 	/* virtual */ bool exists(const char* pathname) const;

utils/messageqcpp/inetstreamsocket.cpp

@@ -436,10 +436,16 @@ const SBS InetStreamSocket::read(const struct ::timespec* timeout, bool* isTimeO
 	uint8_t* msglenp = reinterpret_cast<uint8_t*>(&msglen);
 	size_t mlread = 0;
 	
-	if (readToMagic(msecs, isTimeOut, stats) == false)	//indicates a timeout or EOF
+	bool myIsTimeOut = false;
+	if (readToMagic(msecs, &myIsTimeOut, stats) == false)	//indicates a timeout or EOF
 	{
-		logIoError("InetStreamSocket::read: timeout during readToMagic", 0);
+	    if (!myIsTimeOut)
-		return SBS(new ByteStream(0));
+	        logIoError("InetStreamSocket::read: EOF during readToMagic", 0);
+	    if (isTimeOut)
+	    {
+	        *isTimeOut = myIsTimeOut;
+	    }
+	    return SBS(new ByteStream(0));
 	}
 
 	//FIXME: This seems like a lot of work to read 4 bytes...

writeengine/server/we_ddlcommandproc.cpp

@@ -193,7 +193,7 @@ uint8_t WE_DDLCommandProc::writeSystable(ByteStream& bs, std::string &err)
 
                 t = time(NULL);
                 gmtime_r(&t, &tmp);
-				aDay.year = tmp.tm_year;
+				aDay.year = tmp.tm_year + 1900;
 				aDay.month = tmp.tm_mon+1;
 				aDay.day = tmp.tm_mday;