diff --git a/.gitignore b/.gitignore index c9dc8e53a..ec3d89ef1 100644 --- a/.gitignore +++ b/.gitignore @@ -98,6 +98,7 @@ utils/loggingcpp/messageids.h CPackConfig.cmake CPackSourceConfig.cmake build/columnstore.community.spec +build/releasenum *.rpm install_manifest_libs.txt install_manifest_platform.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index 2df76b54b..edbe69e93 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -36,6 +36,31 @@ ENDIF(NOT CMAKE_BUILD_TYPE) #set( CMAKE_VERBOSE_MAKEFILE on ) +IF (NOT VERSION) + SET (VERSION 1.0.5) +ENDIF() +IF (NOT RELEASE) + SET (RELEASE 1) +ENDIF() + +SET (PACKAGE columnstore) +SET (PACKAGE_NAME columnstore) +SET (PACKAGE_TARNAME columnstore) +SET (PACKAGE_BUGREPORT support@mariadb.com) +SET (PACKAGE_URL "") + +IF (NOT PACKAGE_VERSION) + SET (PACKAGE_VERSION ${VERSION}) +ENDIF() +IF (NOT PACKAGE_RELEASE) + SET (PACKAGE_RELEASE ${RELEASE}) +ENDIF() +SET (PACKAGE_STRING columnstore-${PACKAGE_VERSION}) + + +SET (ENGINE_SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}) +SET (INSTALL_ENGINE "/usr/local/mariadb/columnstore") + # use, i.e. don't skip the full RPATH for the build tree SET(CMAKE_SKIP_BUILD_RPATH FALSE) @@ -43,31 +68,18 @@ SET(CMAKE_SKIP_BUILD_RPATH FALSE) # (but later on when installing) SET(CMAKE_BUILD_WITH_INSTALL_RPATH FALSE) -SET(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib") +SET(CMAKE_INSTALL_RPATH "${INSTALL_ENGINE}/lib") # add the automatically determined parts of the RPATH # which point to directories outside the build tree to the install RPATH SET(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) - # the RPATH to be used when installing, but only if it's not a system directory -LIST(FIND CMAKE_PLATFORM_IMPLICIT_LINK_DIRECTORIES "${CMAKE_INSTALL_PREFIX}/lib" isSystemDir) +LIST(FIND CMAKE_PLATFORM_IMPLICIT_LINK_DIRECTORIES "${INSTALL_ENGINE}/lib" isSystemDir) IF("${isSystemDir}" STREQUAL "-1") - SET(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib") + SET(CMAKE_INSTALL_RPATH "${INSTALL_ENGINE}/lib") ENDIF("${isSystemDir}" STREQUAL "-1") -SET (PACKAGE columnstore) -SET (VERSION 1.0.2) -SET (PACKAGE_NAME columnstore) -SET (PACKAGE_TARNAME columnstore) -SET (PACKAGE_VERSION 1.0.2) -SET (PACKAGE_STRING columnstore-1.0.2) -SET (PACKAGE_BUGREPORT support@mariadb.com) -SET (PACKAGE_URL "") - -SET (ENGINE_SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}) -SET (INSTALL_ENGINE "/usr/local/mariadb/columnstore") - INCLUDE (configureEngine.cmake) CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/build/releasenum.in ${CMAKE_CURRENT_BINARY_DIR}/build/releasenum @ONLY IMMEDIATE) @@ -110,7 +122,7 @@ ENDFOREACH() SET(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -ggdb3 -fno-tree-vectorize -DSAFE_MUTEX -DSAFEMALLOC -DENABLED_DEBUG_SYNC -O0 -Wall -D_DEBUG -DHAVE_CONFIG_H") SET(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -ggdb3 -fno-tree-vectorize -DSAFE_MUTEX -DSAFEMALLOC -DENABLED_DEBUG_SYNC -O0 -Wall -D _DEBUG -DHAVE_CONFIG_H") -SET (ENGINE_LDFLAGS "-Wl,--rpath -Wl,${INSTALL_ENGINE}/lib -Wl,--no-as-needed -Wl,--add-needed") +SET (ENGINE_LDFLAGS "-Wl,--no-as-needed -Wl,--add-needed") FIND_PACKAGE(Boost 1.53.0 REQUIRED COMPONENTS system filesystem thread regex date_time) diff --git a/README b/README index b56720e1e..85722e3c3 100644 --- a/README +++ b/README @@ -1,6 +1,6 @@ This is MariaDB ColumnStore 1.0.4 MariaDB ColumnStore 1.0.4 is the development version of MariaDB ColumnStore. -It is built by porting InfiniDB 4.6.2 on MariaDB 10.1.18 and adding entirely +It is built by porting InfiniDB 4.6.2 on MariaDB 10.1.19 and adding entirely new features not found anywhere else. MariaDB ColumnStore 1.0.4 is an Beta release. This is the first MariaDB diff --git a/README.md b/README.md index 3754dfc2d..62a8a218d 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ -#MariaDB ColumnStore Storage/Execution engine 1.0.4 -MariaDB ColumnStore 1.0.4 is the development version of MariaDB ColumnStore. -It is built by porting InfiniDB 4.6.2 on MariaDB 10.1.18 and adding entirely +#MariaDB ColumnStore Storage/Execution engine 1.0.6 +MariaDB ColumnStore 1.0.6 is the development version of MariaDB ColumnStore. +It is built by porting InfiniDB 4.6.2 on MariaDB 10.1.19 and adding entirely new features not found anywhere else. -#MariaDB ColumnStore 1.0.4 is an Beta release. +#MariaDB ColumnStore 1.0.6 is an GA release. This is the first MariaDB ColumnStore release, not all features planned for the MariaDB ColumnStore 1.0 series are included in this release. diff --git a/build/releasenum b/build/releasenum deleted file mode 100644 index 0aa2a30a8..000000000 --- a/build/releasenum +++ /dev/null @@ -1,2 +0,0 @@ -version=1.0.4 -release=1 diff --git a/build/releasenum.in b/build/releasenum.in index ea20eb5c1..1a05ad69d 100644 --- a/build/releasenum.in +++ b/build/releasenum.in @@ -1,2 +1,2 @@ -version=@CPACK_RPM_PACKAGE_VERSION@ -release=@CPACK_RPM_PACKAGE_RELEASE@ +version=@VERSION@ +release=@RELEASE@ \ No newline at end of file diff --git a/cpackEngineRPM.cmake b/cpackEngineRPM.cmake index 7cee142a8..32552e855 100644 --- a/cpackEngineRPM.cmake +++ b/cpackEngineRPM.cmake @@ -15,10 +15,10 @@ SET(CPACK_PACKAGE_NAME "mariadb-columnstore") SET(ENGINE_ARCH "x86_64") IF (NOT CPACK_RPM_PACKAGE_VERSION) -SET (CPACK_RPM_PACKAGE_VERSION "1.0.0") +SET (CPACK_RPM_PACKAGE_VERSION ${PACKAGE_VERSION}) ENDIF() IF (NOT CPACK_RPM_PACKAGE_RELEASE) -SET (CPACK_RPM_PACKAGE_RELEASE "0") +SET (CPACK_RPM_PACKAGE_RELEASE ${PACKAGE_RELEASE}) ENDIF() SET(CPACK_RPM_PACKAGE_NAME ${CPACK_PACKAGE_NAME}) @@ -194,8 +194,8 @@ SET(CPACK_RPM_platform_USER_FILELIST "/usr/local/mariadb/columnstore/bin/autoConfigure" "/usr/local/mariadb/columnstore/bin/ddlcleanup" "/usr/local/mariadb/columnstore/bin/idbmeminfo" -"/usr/local/mariadb/columnstore/bin/IDBInstanceCmds.sh" -"/usr/local/mariadb/columnstore/bin/IDBVolumeCmds.sh" +"/usr/local/mariadb/columnstore/bin/MCSInstanceCmds.sh" +"/usr/local/mariadb/columnstore/bin/MCSVolumeCmds.sh" "/usr/local/mariadb/columnstore/bin/binary_installer.sh" "/usr/local/mariadb/columnstore/bin/myCnf-include-args.text" "/usr/local/mariadb/columnstore/bin/myCnf-exclude-args.text" @@ -318,6 +318,18 @@ SET(CPACK_RPM_storage-engine_USER_FILELIST "/usr/local/mariadb/columnstore/lib/libudf_mysql.so.1.0.0" "/usr/local/mariadb/columnstore/lib/libudf_mysql.so.1" "/usr/local/mariadb/columnstore/lib/libudf_mysql.so" +"/usr/local/mariadb/columnstore/lib/is_columnstore_columns.so" +"/usr/local/mariadb/columnstore/lib/is_columnstore_columns.so.1" +"/usr/local/mariadb/columnstore/lib/is_columnstore_columns.so.1.0.0" +"/usr/local/mariadb/columnstore/lib/is_columnstore_extents.so" +"/usr/local/mariadb/columnstore/lib/is_columnstore_extents.so.1" +"/usr/local/mariadb/columnstore/lib/is_columnstore_extents.so.1.0.0" +"/usr/local/mariadb/columnstore/lib/is_columnstore_tables.so" +"/usr/local/mariadb/columnstore/lib/is_columnstore_tables.so.1" +"/usr/local/mariadb/columnstore/lib/is_columnstore_tables.so.1.0.0" +"/usr/local/mariadb/columnstore/lib/is_columnstore_files.so" +"/usr/local/mariadb/columnstore/lib/is_columnstore_files.so.1" +"/usr/local/mariadb/columnstore/lib/is_columnstore_files.so.1.0.0" "/usr/local/mariadb/columnstore/mysql/mysql-Columnstore" "/usr/local/mariadb/columnstore/mysql/install_calpont_mysql.sh" "/usr/local/mariadb/columnstore/mysql/syscatalog_mysql.sql" @@ -326,6 +338,7 @@ SET(CPACK_RPM_storage-engine_USER_FILELIST "/usr/local/mariadb/columnstore/mysql/calsetuserpriority.sql" "/usr/local/mariadb/columnstore/mysql/calremoveuserpriority.sql" "/usr/local/mariadb/columnstore/mysql/calshowprocesslist.sql" +"/usr/local/mariadb/columnstore/mysql/columnstore_info.sql" ${ignored}) diff --git a/dbcon/execplan/calpontsystemcatalog.cpp b/dbcon/execplan/calpontsystemcatalog.cpp index ecec4fcf9..635927c7b 100644 --- a/dbcon/execplan/calpontsystemcatalog.cpp +++ b/dbcon/execplan/calpontsystemcatalog.cpp @@ -2599,6 +2599,7 @@ const vector< pair > SimpleColumn *c1 = new SimpleColumn(CALPONT_SCHEMA+"."+SYSTABLE_TABLE+"."+TABLENAME_COL, fSessionID); SimpleColumn *c2 = new SimpleColumn(CALPONT_SCHEMA+"."+SYSTABLE_TABLE+"."+SCHEMA_COL, fSessionID); SimpleColumn *c3 = new SimpleColumn(CALPONT_SCHEMA+"."+SYSTABLE_TABLE+"."+OBJECTID_COL, fSessionID); + SimpleColumn *c4 = new SimpleColumn(CALPONT_SCHEMA+"."+SYSTABLE_TABLE+"."+CREATEDATE_COL, fSessionID); SRCP srcp; srcp.reset(c1); @@ -2607,6 +2608,8 @@ const vector< pair > colMap.insert(CMVT_(CALPONT_SCHEMA+"."+SYSTABLE_TABLE+"."+SCHEMA_COL, srcp)); srcp.reset(c3); colMap.insert(CMVT_(CALPONT_SCHEMA+"."+SYSTABLE_TABLE+"."+OBJECTID_COL, srcp)); + srcp.reset(c4); + colMap.insert(CMVT_(CALPONT_SCHEMA+"."+SYSTABLE_TABLE+"."+CREATEDATE_COL, srcp)); csep.columnMapNonStatic(colMap); srcp.reset(c1->clone()); @@ -2653,6 +2656,14 @@ const vector< pair > tables[i].second.schema = (*it)->GetStringData(i); } } + for (it = sysDataList.begin(); it != sysDataList.end(); it++) + { + if ((*it)->ColumnOID() == c4->oid()) + { + for (int i = 0; i < (*it)->dataCount(); i++) + tables[i].second.create_date = (*it)->GetData(i); + } + } for (it = sysDataList.begin(); it != sysDataList.end(); it++) { diff --git a/dbcon/execplan/calpontsystemcatalog.h b/dbcon/execplan/calpontsystemcatalog.h index 42b721d65..2936b2daa 100644 --- a/dbcon/execplan/calpontsystemcatalog.h +++ b/dbcon/execplan/calpontsystemcatalog.h @@ -431,6 +431,7 @@ public: TableName(const TableAliasName& tan): schema(tan.schema), table(tan.table) {} std::string schema; std::string table; + int64_t create_date; bool operator<(const TableName& rhs) const; bool operator>=(const TableName& rhs) const { return !(*this < rhs); } bool operator==(const TableName& rhs) const diff --git a/dbcon/mysql/CMakeLists.txt b/dbcon/mysql/CMakeLists.txt index 7dde5b208..d478843b0 100644 --- a/dbcon/mysql/CMakeLists.txt +++ b/dbcon/mysql/CMakeLists.txt @@ -17,7 +17,7 @@ SET ( libcalmysql_SRCS ha_view.cpp sm.cpp ha_window_function.cpp ha_calpont_partition.cpp - ha_pseudocolumn.cpp ) + ha_pseudocolumn.cpp) add_definitions(-DMYSQL_DYNAMIC_PLUGIN) @@ -29,12 +29,58 @@ target_link_libraries(calmysql ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${NETSNMP_ set_target_properties(calmysql PROPERTIES VERSION 1.0.0 SOVERSION 1) -install(TARGETS calmysql DESTINATION ${ENGINE_LIBDIR} COMPONENT storage-engine) +SET ( is_columnstore_tables_SRCS + is_columnstore_tables.cpp + ) +add_library(is_columnstore_tables SHARED ${is_columnstore_tables_SRCS}) + +target_link_libraries(is_columnstore_tables ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} threadpool) + +# Don't prepend .so file with 'lib' +set_target_properties(is_columnstore_tables PROPERTIES PREFIX "") +set_target_properties(is_columnstore_tables PROPERTIES VERSION 1.0.0 SOVERSION 1) + +SET ( is_columnstore_columns_SRCS + is_columnstore_columns.cpp + ) +add_library(is_columnstore_columns SHARED ${is_columnstore_columns_SRCS}) + +target_link_libraries(is_columnstore_columns ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} threadpool) + +# Don't prepend .so file with 'lib' +set_target_properties(is_columnstore_columns PROPERTIES PREFIX "") +set_target_properties(is_columnstore_columns PROPERTIES VERSION 1.0.0 SOVERSION 1) + +SET ( is_columnstore_extents_SRCS + is_columnstore_extents.cpp + ) +add_library(is_columnstore_extents SHARED ${is_columnstore_extents_SRCS}) + +target_link_libraries(is_columnstore_extents ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} threadpool) + +# Don't prepend .so file with 'lib' +set_target_properties(is_columnstore_extents PROPERTIES PREFIX "") +set_target_properties(is_columnstore_extents PROPERTIES VERSION 1.0.0 SOVERSION 1) + +SET ( is_columnstore_files_SRCS + is_columnstore_files.cpp + ) +add_library(is_columnstore_files SHARED ${is_columnstore_files_SRCS}) + +target_link_libraries(is_columnstore_files ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} threadpool) + +# Don't prepend .so file with 'lib' +set_target_properties(is_columnstore_files PROPERTIES PREFIX "") +set_target_properties(is_columnstore_files PROPERTIES VERSION 1.0.0 SOVERSION 1) + + +install(TARGETS calmysql is_columnstore_tables is_columnstore_columns is_columnstore_extents is_columnstore_files DESTINATION ${ENGINE_LIBDIR} COMPONENT storage-engine) install(FILES syscatalog_mysql.sql dumpcat_mysql.sql calsetuserpriority.sql calremoveuserpriority.sql calshowprocesslist.sql + columnstore_info.sql my.cnf DESTINATION ${ENGINE_MYSQLDIR} COMPONENT storage-engine) install(PROGRAMS install_calpont_mysql.sh mysql-Columnstore dumpcat.pl diff --git a/dbcon/mysql/columnstore_info.sql b/dbcon/mysql/columnstore_info.sql new file mode 100644 index 000000000..d900c99b0 --- /dev/null +++ b/dbcon/mysql/columnstore_info.sql @@ -0,0 +1,50 @@ +CREATE DATABASE columnstore_info; +USE columnstore_info; + +DELIMITER // +CREATE FUNCTION `format_filesize`(filesize FLOAT) RETURNS varchar(20) CHARSET utf8 DETERMINISTIC +BEGIN + +DECLARE n INT DEFAULT 1; + +LOOP + IF filesize < 1024 THEN + RETURN concat(round(filesize, 2), ' ', elt(n, 'Bytes', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB', 'BB')); + END IF; + SET filesize = filesize / 1024; + SET n = n + 1; +END LOOP; + +END // + + +CREATE PROCEDURE total_usage () +BEGIN + SELECT + (SELECT format_filesize(sum(data_size)) TOTAL_DATA_SIZE FROM INFORMATION_SCHEMA.COLUMNSTORE_EXTENTS) TOTAL_DATA_SIZE, + (SELECT format_filesize(sum(file_size)) TOTAL_DISK_USAGE FROM INFORMATION_SCHEMA.COLUMNSTORE_FILES) TOTAL_DISK_USAGE; +END // + +CREATE PROCEDURE table_usage (IN t_name char(64)) +BEGIN + IF t_name IS NOT NULL THEN + SELECT TABLE_NAME, format_filesize(sum(cf.file_size)) DATA_DISK_USAGE, format_filesize(sum(IFNULL(ccf.file_size, 0))) DICT_DISK_USAGE, format_filesize(sum(cf.file_size) + sum(IFNULL(ccf.file_size, 0))) TOTAL_USAGE FROM INFORMATION_SCHEMA.COLUMNSTORE_COLUMNS cc +JOIN INFORMATION_SCHEMA.COLUMNSTORE_FILES cf ON cc.object_id = cf.object_id +LEFT JOIN INFORMATION_SCHEMA.COLUMNSTORE_FILES ccf ON cc.dictionary_object_id = ccf.object_id +WHERE table_name = t_name GROUP BY table_name; + ELSE + SELECT TABLE_NAME, format_filesize(sum(cf.file_size)) DATA_DISK_USAGE, format_filesize(sum(IFNULL(ccf.file_size, 0))) DICT_DISK_USAGE, format_filesize(sum(cf.file_size) + sum(IFNULL(ccf.file_size, 0))) TOTAL_USAGE FROM INFORMATION_SCHEMA.COLUMNSTORE_COLUMNS cc +JOIN INFORMATION_SCHEMA.COLUMNSTORE_FILES cf ON cc.object_id = cf.object_id +LEFT JOIN INFORMATION_SCHEMA.COLUMNSTORE_FILES ccf ON cc.dictionary_object_id = ccf.object_id +GROUP BY table_name; + END IF; +END // + +CREATE PROCEDURE compression_ratio() +BEGIN +SELECT CONCAT(((sum(compressed_data_size) / sum(data_size)) * 100), '%') COMPRESSION_RATIO FROM INFORMATION_SCHEMA.COLUMNSTORE_EXTENTS ce +JOIN INFORMATION_SCHEMA.COLUMNSTORE_FILES cf ON ce.object_id = cf.object_id +WHERE compressed_data_size IS NOT NULL; +END // + +DELIMITER ; diff --git a/dbcon/mysql/install_calpont_mysql.sh b/dbcon/mysql/install_calpont_mysql.sh index 5b2b981df..9b55f9448 100755 --- a/dbcon/mysql/install_calpont_mysql.sh +++ b/dbcon/mysql/install_calpont_mysql.sh @@ -30,6 +30,10 @@ df=$installdir/mysql/my.cnf $installdir/mysql/bin/mysql --defaults-file=$df --force --user=root $pwprompt mysql 2>/tmp/mysql_install.log </d $installdir/mysql/bin/mysql --defaults-file=$df --user=root $pwprompt mysql 2>/dev/null <$installdir/mysql/calsetuserpriority.sql $installdir/mysql/bin/mysql --defaults-file=$df --user=root $pwprompt mysql 2>/dev/null <$installdir/mysql/calremoveuserpriority.sql $installdir/mysql/bin/mysql --defaults-file=$df --user=root $pwprompt mysql 2>/dev/null <$installdir/mysql/calshowprocesslist.sql +$installdir/mysql/bin/mysql --defaults-file=$df --user=root $pwprompt mysql 2>/dev/null <$installdir/mysql/columnstore_info.sql sed -i 's/infinidb_compression_type=1/infinidb_compression_type=2/' $installdir/mysql/my.cnf >/dev/null 2>&1 diff --git a/dbcon/mysql/is_columnstore_columns.cpp b/dbcon/mysql/is_columnstore_columns.cpp new file mode 100644 index 000000000..ea9e80aa3 --- /dev/null +++ b/dbcon/mysql/is_columnstore_columns.cpp @@ -0,0 +1,187 @@ +/* c-basic-offset: 4; tab-width: 4; indent-tabs-mode: nil + * vi: set shiftwidth=4 tabstop=4 expandtab: + * :indentSize=4:tabSize=4:noTabs=true: + * + * Copyright (C) 2016 MariaDB Corporaton + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; version 2 of + * the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301, USA. + */ + +#include "idb_mysql.h" +#include +#include + +#include +#include "calpontsystemcatalog.h" +#include "dataconvert.h" + + +// Required declaration as it isn't in a MairaDB include +bool schema_table_store_record(THD *thd, TABLE *table); + +ST_FIELD_INFO is_columnstore_columns_fields[] = +{ + {"TABLE_SCHEMA", 64, MYSQL_TYPE_STRING, 0, 0, 0, 0}, + {"TABLE_NAME", 64, MYSQL_TYPE_STRING, 0, 0, 0, 0}, + {"COLUMN_NAME", 64, MYSQL_TYPE_STRING, 0, 0, 0, 0}, + {"OBJECT_ID", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0}, + {"DICTIONARY_OBJECT_ID", 11, MYSQL_TYPE_LONG, 0, MY_I_S_MAYBE_NULL, 0, 0}, + {"LIST_OBJECT_ID", 11, MYSQL_TYPE_LONG, 0, MY_I_S_MAYBE_NULL, 0, 0}, + {"TREE_OBJECT_ID", 11, MYSQL_TYPE_LONG, 0, MY_I_S_MAYBE_NULL, 0, 0}, + {"DATA_TYPE", 64, MYSQL_TYPE_STRING, 0, 0, 0, 0}, + {"COLUMN_LENGTH", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0}, + {"COLUMN_POSITION", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0}, + {"COLUMN_DEFAULT", 0, MYSQL_TYPE_LONG_BLOB, 0, MY_I_S_MAYBE_NULL, 0, 0}, + {"IS_NULLABLE", 1, MYSQL_TYPE_TINY, 0, 0, 0, 0}, + {"NUMERIC_PRECISION", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0}, + {"NUMERIC_SCALE", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0}, + {"IS_AUTOINCREMENT", 1, MYSQL_TYPE_TINY, 0, 0, 0, 0}, + {"COMPRESSION_TYPE", 64, MYSQL_TYPE_STRING, 0, 0, 0, 0}, + {0, 0, MYSQL_TYPE_NULL, 0, 0, 0, 0} + +}; + +static int is_columnstore_columns_fill(THD *thd, TABLE_LIST *tables, COND *cond) +{ + CHARSET_INFO *cs = system_charset_info; + TABLE *table = tables->table; + + boost::shared_ptr systemCatalogPtr = + execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(execplan::CalpontSystemCatalog::idb_tid2sid(thd->thread_id)); + + const std::vector< std::pair > catalog_tables + = systemCatalogPtr->getTables(); + + for (std::vector >::const_iterator it = catalog_tables.begin(); + it != catalog_tables.end(); ++it) + { + execplan::CalpontSystemCatalog::RIDList column_rid_list = systemCatalogPtr->columnRIDs((*it).second); + for (size_t col_num = 0; col_num < column_rid_list.size(); col_num++) + { + execplan::CalpontSystemCatalog::TableColName tcn = systemCatalogPtr->colName(column_rid_list[col_num].objnum); + execplan::CalpontSystemCatalog::ColType ct = systemCatalogPtr->colType(column_rid_list[col_num].objnum); + + table->field[0]->store(tcn.schema.c_str(), tcn.schema.length(), cs); + table->field[1]->store(tcn.table.c_str(), tcn.table.length(), cs); + table->field[2]->store(tcn.column.c_str(), tcn.column.length(), cs); + table->field[3]->store(column_rid_list[col_num].objnum); + if (ct.ddn.dictOID == std::numeric_limits::min()) + { + table->field[4]->set_null(); + } + else + { + table->field[4]->set_notnull(); + table->field[4]->store(ct.ddn.dictOID); + } + if (ct.ddn.listOID == std::numeric_limits::min()) + { + table->field[5]->set_null(); + } + else + { + table->field[5]->set_notnull(); + table->field[5]->store(ct.ddn.listOID); + } + if (ct.ddn.treeOID == std::numeric_limits::min()) + { + table->field[6]->set_null(); + } + else + { + table->field[6]->set_notnull(); + table->field[6]->store(ct.ddn.treeOID); + } + std::string data_type = execplan::colDataTypeToString(ct.colDataType); + table->field[7]->store(data_type.c_str(), data_type.length(), cs); + table->field[8]->store(ct.colWidth); + table->field[9]->store(ct.colPosition); + if (ct.defaultValue.empty()) + { + table->field[10]->set_null(); + } + else + { + table->field[10]->set_notnull(); + table->field[10]->store(ct.defaultValue.c_str(), ct.defaultValue.length(), cs); + } + table->field[11]->store(ct.autoincrement); + table->field[12]->store(ct.precision); + table->field[13]->store(ct.scale); + if (ct.constraintType != execplan::CalpontSystemCatalog::NOTNULL_CONSTRAINT) + { + table->field[14]->store(true); + } + else + { + table->field[14]->store(false); + } + + std::string compression_type; + switch (ct.compressionType) + { + case 0: + compression_type = "None"; + break; + case 2: + compression_type = "Snappy"; + break; + default: + compression_type = "Unknown"; + break; + } + table->field[15]->store(compression_type.c_str(), compression_type.length(), cs); + if (schema_table_store_record(thd, table)) + return 1; + } + } + + + + return 0; +} + +static int is_columnstore_columns_plugin_init(void *p) +{ + ST_SCHEMA_TABLE *schema = (ST_SCHEMA_TABLE*) p; + schema->fields_info = is_columnstore_columns_fields; + schema->fill_table = is_columnstore_columns_fill; + return 0; +} + +static struct st_mysql_information_schema is_columnstore_columns_plugin_version = +{ MYSQL_INFORMATION_SCHEMA_INTERFACE_VERSION }; + +maria_declare_plugin(is_columnstore_columns_plugin) +{ + MYSQL_INFORMATION_SCHEMA_PLUGIN, + &is_columnstore_columns_plugin_version, + "COLUMNSTORE_COLUMNS", + "MariaDB Corporaton", + "An information schema plugin to list ColumnStore columns", + PLUGIN_LICENSE_GPL, + is_columnstore_columns_plugin_init, + //is_columnstore_tables_plugin_deinit, + NULL, + 0x0100, + NULL, + NULL, + "1.0", + MariaDB_PLUGIN_MATURITY_STABLE +} +maria_declare_plugin_end; + + diff --git a/dbcon/mysql/is_columnstore_extents.cpp b/dbcon/mysql/is_columnstore_extents.cpp new file mode 100644 index 000000000..1d1098eda --- /dev/null +++ b/dbcon/mysql/is_columnstore_extents.cpp @@ -0,0 +1,202 @@ +/* c-basic-offset: 4; tab-width: 4; indent-tabs-mode: nil + * vi: set shiftwidth=4 tabstop=4 expandtab: + * :indentSize=4:tabSize=4:noTabs=true: + * + * Copyright (C) 2016 MariaDB Corporaton + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; version 2 of + * the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301, USA. + */ + +#include "idb_mysql.h" +#include +#include + +#include "dbrm.h" +#include "objectidmanager.h" + + +// Required declaration as it isn't in a MairaDB include +bool schema_table_store_record(THD *thd, TABLE *table); + +ST_FIELD_INFO is_columnstore_extents_fields[] = +{ + {"OBJECT_ID", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0}, // 0 + {"OBJECT_TYPE", 64, MYSQL_TYPE_STRING, 0, 0, 0, 0}, // 1 + {"LOGICAL_BLOCK_START", 19, MYSQL_TYPE_LONGLONG, 0, 0, 0, 0}, // 2 + {"LOGICAL_BLOCK_END", 19, MYSQL_TYPE_LONGLONG, 0, 0, 0, 0}, // 3 + {"MIN_VALUE", 19, MYSQL_TYPE_LONGLONG, 0, MY_I_S_MAYBE_NULL, 0, 0}, // 4 + {"MAX_VALUE", 19, MYSQL_TYPE_LONGLONG, 0, MY_I_S_MAYBE_NULL, 0, 0}, // 5 + {"WIDTH", 1, MYSQL_TYPE_TINY, 0, 0, 0, 0}, // 6 + {"DBROOT", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0}, // 7 + {"PARTITION_ID", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0}, // 8 + {"SEGMENT_ID", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0}, // 9 + {"BLOCK_OFFSET", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0}, // 10 + {"MAX_BLOCKS", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0}, // 11 + {"HIGH_WATER_MARK", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0}, // 12 + {"STATE", 64, MYSQL_TYPE_STRING, 0, 0, 0, 0}, // 13 + {"STATUS", 64, MYSQL_TYPE_STRING, 0, 0, 0, 0}, // 14 + {"DATA_SIZE", 19, MYSQL_TYPE_LONGLONG, 0, 0, 0, 0}, // 15 + {0, 0, MYSQL_TYPE_NULL, 0, 0, 0, 0} +}; + +static int is_columnstore_extents_fill(THD *thd, TABLE_LIST *tables, COND *cond) +{ + CHARSET_INFO *cs = system_charset_info; + TABLE *table = tables->table; + std::vector entries; + std::vector::iterator iter; + std::vector::iterator end; + + BRM::DBRM *emp = new BRM::DBRM(); + if (!emp || !emp->isDBRMReady()) + { + return 1; + } + + execplan::ObjectIDManager oidm; + BRM::OID_t MaxOID = oidm.size(); + + for(BRM::OID_t oid = 3000; oid <= MaxOID; oid++) + { + emp->getExtents(oid, entries, false, false, true); + if (entries.size() == 0) + continue; + + iter = entries.begin(); + end = entries.end(); + while (iter != end) + { + table->field[0]->store(oid); + if (iter->colWid > 0) + { + table->field[1]->store("Column", strlen("Column"), cs); + if (iter->partition.cprange.lo_val == std::numeric_limits::max() || + iter->partition.cprange.lo_val <= (std::numeric_limits::min() + 2)) + { + table->field[4]->set_null(); + } + else + { + table->field[4]->set_notnull(); + table->field[4]->store(iter->partition.cprange.lo_val); + } + if (iter->partition.cprange.hi_val == std::numeric_limits::max() || + iter->partition.cprange.hi_val <= (std::numeric_limits::min() + 2)) + { + table->field[5]->set_null(); + } + else + { + table->field[5]->set_notnull(); + table->field[5]->store(iter->partition.cprange.hi_val); + } + table->field[6]->store(iter->colWid); + + } + else + { + table->field[1]->store("Dictionary", strlen("Dictionary"), cs); + table->field[4]->set_null(); + table->field[5]->set_null(); + table->field[6]->store(8192); + } + table->field[2]->store(iter->range.start); + table->field[3]->store(iter->range.start + (iter->range.size * 1024) - 1); + + table->field[7]->store(iter->dbRoot); + table->field[8]->store(iter->partitionNum); + table->field[9]->store(iter->segmentNum); + table->field[10]->store(iter->blockOffset); + table->field[11]->store(iter->range.size * 1024); + table->field[12]->store(iter->HWM); + + switch (iter->partition.cprange.isValid) + { + case 0: + table->field[13]->store("Invalid", strlen("Invalid"), cs); + break; + case 1: + table->field[13]->store("Updating", strlen("Updating"), cs); + break; + case 2: + table->field[13]->store("Valid", strlen("Valid"), cs); + break; + default: + table->field[13]->store("Unknown", strlen("Unknown"), cs); + break; + } + switch (iter->status) + { + case BRM::EXTENTAVAILABLE: + table->field[14]->store("Available", strlen("Available"), cs); + break; + case BRM::EXTENTUNAVAILABLE: + table->field[14]->store("Unavailable", strlen("Unavailable"), cs); + break; + case BRM::EXTENTOUTOFSERVICE: + table->field[14]->store("Out of service", strlen("Out of service"), cs); + break; + default: + table->field[14]->store("Unknown", strlen("Unknown"), cs); + } + table->field[15]->store((iter->HWM - iter->blockOffset + 1) * 8192); + + if (schema_table_store_record(thd, table)) + { + delete emp; + return 1; + } + + iter++; + + } + } + + delete emp; + return 0; +} + +static int is_columnstore_extents_plugin_init(void *p) +{ + ST_SCHEMA_TABLE *schema = (ST_SCHEMA_TABLE*) p; + schema->fields_info = is_columnstore_extents_fields; + schema->fill_table = is_columnstore_extents_fill; + return 0; +} + +static struct st_mysql_information_schema is_columnstore_extents_plugin_version = +{ MYSQL_INFORMATION_SCHEMA_INTERFACE_VERSION }; + +maria_declare_plugin(is_columnstore_extents_plugin) +{ + MYSQL_INFORMATION_SCHEMA_PLUGIN, + &is_columnstore_extents_plugin_version, + "COLUMNSTORE_EXTENTS", + "MariaDB Corporaton", + "An information schema plugin to list ColumnStore extents", + PLUGIN_LICENSE_GPL, + is_columnstore_extents_plugin_init, + //is_columnstore_extents_plugin_deinit, + NULL, + 0x0100, + NULL, + NULL, + "1.0", + MariaDB_PLUGIN_MATURITY_STABLE +} +maria_declare_plugin_end; + + diff --git a/dbcon/mysql/is_columnstore_files.cpp b/dbcon/mysql/is_columnstore_files.cpp new file mode 100644 index 000000000..b39be0ee6 --- /dev/null +++ b/dbcon/mysql/is_columnstore_files.cpp @@ -0,0 +1,163 @@ +/* c-basic-offset: 4; tab-width: 4; indent-tabs-mode: nil + * vi: set shiftwidth=4 tabstop=4 expandtab: + * :indentSize=4:tabSize=4:noTabs=true: + * + * Copyright (C) 2016 MariaDB Corporaton + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; version 2 of + * the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301, USA. + */ + +#include "idb_mysql.h" + +#include "dbrm.h" +#include "objectidmanager.h" +#include "we_convertor.h" +#include "we_define.h" +#include "IDBPolicy.h" +#include "we_config.h" +#include "we_brm.h" + +// Required declaration as it isn't in a MairaDB include +bool schema_table_store_record(THD *thd, TABLE *table); + +ST_FIELD_INFO is_columnstore_files_fields[] = +{ + {"OBJECT_ID", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0}, + {"SEGMENT_ID", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0}, + {"PARTITION_ID", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0}, + {"FILENAME", 1024, MYSQL_TYPE_STRING, 0, 0, 0, 0}, + {"FILE_SIZE", 19, MYSQL_TYPE_LONGLONG, 0, MY_I_S_MAYBE_NULL, 0, 0}, + {"COMPRESSED_DATA_SIZE", 19, MYSQL_TYPE_LONGLONG, 0, MY_I_S_MAYBE_NULL, 0, 0}, + {0, 0, MYSQL_TYPE_NULL, 0, 0, 0, 0} +}; + + +static bool is_columnstore_files_get_entries(THD *thd, TABLE_LIST *tables, BRM::OID_t oid, std::vector &entries) +{ + CHARSET_INFO *cs = system_charset_info; + TABLE *table = tables->table; + + char oidDirName[WriteEngine::FILE_NAME_SIZE]; + char fullFileName[WriteEngine::FILE_NAME_SIZE]; + char dbDir[WriteEngine::MAX_DB_DIR_LEVEL][WriteEngine::MAX_DB_DIR_NAME_SIZE]; + WriteEngine::Config config; + config.initConfigCache(); + + std::vector::const_iterator iter = entries.begin(); + while ( iter != entries.end() ) //organize extents into files + { + // Don't include files more than once at different block offsets + if (iter->blockOffset > 0) + { + iter++; + continue; + } + table->field[0]->store(oid); + table->field[1]->store(iter->segmentNum); + table->field[2]->store(iter->partitionNum); + + WriteEngine::Convertor::oid2FileName(oid, oidDirName, dbDir, iter->partitionNum, iter->segmentNum); + snprintf(fullFileName, WriteEngine::FILE_NAME_SIZE, "%s/%s", WriteEngine::Config::getDBRootByNum(iter->dbRoot).c_str(), oidDirName); + table->field[3]->store(fullFileName, strlen(fullFileName), cs); + + if (idbdatafile::IDBPolicy::exists(fullFileName)) + { + table->field[4]->set_notnull(); + table->field[4]->store(idbdatafile::IDBPolicy::size(fullFileName)); + off64_t comp_size; + if ((comp_size = idbdatafile::IDBPolicy::compressedSize(fullFileName)) != -1) + { + table->field[5]->set_notnull(); + table->field[5]->store(comp_size); + } + else + { + table->field[5]->set_null(); + } + } + else + { + table->field[4]->set_null(); + table->field[5]->set_null(); + } + + if (schema_table_store_record(thd, table)) + return 1; + iter++; + } + return 0; +} + +static int is_columnstore_files_fill(THD *thd, TABLE_LIST *tables, COND *cond) +{ + BRM::DBRM *emp = new BRM::DBRM(); + std::vector entries; + + if (!emp || !emp->isDBRMReady()) + { + return 1; + } + + execplan::ObjectIDManager oidm; + BRM::OID_t MaxOID = oidm.size(); + + for(BRM::OID_t oid = 3000; oid <= MaxOID; oid++) + { + emp->getExtents(oid, entries, false, false, true); + if (entries.size() == 0) + continue; + + if (is_columnstore_files_get_entries(thd, tables, oid, entries)) + { + delete emp; + return 1; + } + + } + + delete emp; + return 0; +} + +static int is_columnstore_files_plugin_init(void *p) +{ + ST_SCHEMA_TABLE *schema = (ST_SCHEMA_TABLE*) p; + schema->fields_info = is_columnstore_files_fields; + schema->fill_table = is_columnstore_files_fill; + return 0; +} + +static struct st_mysql_information_schema is_columnstore_files_plugin_version = +{ MYSQL_INFORMATION_SCHEMA_INTERFACE_VERSION }; + +maria_declare_plugin(is_columnstore_files_plugin) +{ + MYSQL_INFORMATION_SCHEMA_PLUGIN, + &is_columnstore_files_plugin_version, + "COLUMNSTORE_FILES", + "MariaDB Corporaton", + "An information schema plugin to list ColumnStore filess", + PLUGIN_LICENSE_GPL, + is_columnstore_files_plugin_init, + //is_columnstore_files_plugin_deinit, + NULL, + 0x0100, + NULL, + NULL, + "1.0", + MariaDB_PLUGIN_MATURITY_STABLE +} +maria_declare_plugin_end; diff --git a/dbcon/mysql/is_columnstore_tables.cpp b/dbcon/mysql/is_columnstore_tables.cpp new file mode 100644 index 000000000..5b3bc7fc6 --- /dev/null +++ b/dbcon/mysql/is_columnstore_tables.cpp @@ -0,0 +1,113 @@ +/* c-basic-offset: 4; tab-width: 4; indent-tabs-mode: nil + * vi: set shiftwidth=4 tabstop=4 expandtab: + * :indentSize=4:tabSize=4:noTabs=true: + * + * Copyright (C) 2016 MariaDB Corporaton + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; version 2 of + * the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301, USA. + */ + +#include "idb_mysql.h" +#include + +#include +#include "calpontsystemcatalog.h" +#include "dataconvert.h" + + +// Required declaration as it isn't in a MairaDB include +bool schema_table_store_record(THD *thd, TABLE *table); + +ST_FIELD_INFO is_columnstore_tables_fields[] = +{ + {"TABLE_SCHEMA", 64, MYSQL_TYPE_STRING, 0, 0, 0, 0}, + {"TABLE_NAME", 64, MYSQL_TYPE_STRING, 0, 0, 0, 0}, + {"OBJECT_ID", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0}, + {"CREATION_DATE", 0, MYSQL_TYPE_DATE, 0, 0, 0, 0}, + {"COLUMN_COUNT", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0}, + {"AUTOINCREMENT", 11, MYSQL_TYPE_LONG, 0, MY_I_S_MAYBE_NULL, 0, 0}, + {0, 0, MYSQL_TYPE_NULL, 0, 0, 0, 0} +}; + +static int is_columnstore_tables_fill(THD *thd, TABLE_LIST *tables, COND *cond) +{ + CHARSET_INFO *cs = system_charset_info; + TABLE *table = tables->table; + + boost::shared_ptr systemCatalogPtr = + execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(execplan::CalpontSystemCatalog::idb_tid2sid(thd->thread_id)); + + const std::vector< std::pair > catalog_tables + = systemCatalogPtr->getTables(); + + for (std::vector >::const_iterator it = catalog_tables.begin(); + it != catalog_tables.end(); ++it) + { + execplan::CalpontSystemCatalog::TableInfo tb_info = systemCatalogPtr->tableInfo((*it).second); + std::string create_date = dataconvert::DataConvert::dateToString((*it).second.create_date); + table->field[0]->store((*it).second.schema.c_str(), (*it).second.schema.length(), cs); + table->field[1]->store((*it).second.table.c_str(), (*it).second.table.length(), cs); + table->field[2]->store((*it).first); + table->field[3]->store(create_date.c_str(), create_date.length(), cs); + table->field[4]->store(tb_info.numOfCols); + if (tb_info.tablewithautoincr) + { + table->field[5]->set_notnull(); + table->field[5]->store(systemCatalogPtr->nextAutoIncrValue((*it).second)); + } + else + { + table->field[5]->set_null(); + } + table->field[5]->store(tb_info.tablewithautoincr); + if (schema_table_store_record(thd, table)) + return 1; + } + + return 0; +} + +static int is_columnstore_tables_plugin_init(void *p) +{ + ST_SCHEMA_TABLE *schema = (ST_SCHEMA_TABLE*) p; + schema->fields_info = is_columnstore_tables_fields; + schema->fill_table = is_columnstore_tables_fill; + return 0; +} + +static struct st_mysql_information_schema is_columnstore_tables_plugin_version = +{ MYSQL_INFORMATION_SCHEMA_INTERFACE_VERSION }; + +maria_declare_plugin(is_columnstore_tables_plugin) +{ + MYSQL_INFORMATION_SCHEMA_PLUGIN, + &is_columnstore_tables_plugin_version, + "COLUMNSTORE_TABLES", + "MariaDB Corporaton", + "An information schema plugin to list ColumnStore tables", + PLUGIN_LICENSE_GPL, + is_columnstore_tables_plugin_init, + //is_columnstore_tables_plugin_deinit, + NULL, + 0x0100, + NULL, + NULL, + "1.0", + MariaDB_PLUGIN_MATURITY_STABLE +} +maria_declare_plugin_end; + + diff --git a/oam/cloud/CMakeLists.txt b/oam/cloud/CMakeLists.txt index 6e6469b41..8c18ca72c 100644 --- a/oam/cloud/CMakeLists.txt +++ b/oam/cloud/CMakeLists.txt @@ -1,2 +1,2 @@ -install(PROGRAMS IDBInstanceCmds.sh IDBVolumeCmds.sh DESTINATION ${ENGINE_BINDIR} COMPONENT platform) +install(PROGRAMS MCSInstanceCmds.sh MCSVolumeCmds.sh DESTINATION ${ENGINE_BINDIR} COMPONENT platform) diff --git a/oam/cloud/MCSInstanceCmds.sh b/oam/cloud/MCSInstanceCmds.sh new file mode 100644 index 000000000..225fcbf69 --- /dev/null +++ b/oam/cloud/MCSInstanceCmds.sh @@ -0,0 +1,574 @@ +#! /bin/sh +# MCSInstanceCmds.sh +# get-local-instance-ID, get-zone, getPrivateIP from a Cloud environment +# +# 1. Amazon EC2 + +prefix=/usr/local + +#check command +if [ "$1" = "" ]; then + echo "Enter Command Name: {launchInstance|getInstance|getZone|getPrivateIP|getKey|getAMI|getType|terminateInstance|startInstance|assignElasticIP|deassignElasticIP|getProfile|stopInstance|getGroup|getSubnet} +}" + exit 1 +fi + +if [ "$1" = "getPrivateIP" ]; then + if [ "$2" = "" ]; then + echo "Enter Instance Name" + exit 1 + fi + instanceName="$2" +fi + +if [ "$1" = "launchInstance" ]; then + if [ "$2" = "" ]; then + IPaddress="unassigned" + else + IPaddress="$2" + fi + if [ "$3" = "" ]; then + instanceType="unassigned" + else + instanceType="$3" + fi + if [ "$4" = "" ]; then + group="unassigned" + else + group="$4" + fi +fi + +if [ "$1" = "terminateInstance" ]; then + if [ "$2" = "" ]; then + echo "Enter Instance Name" + exit 1 + fi + instanceName="$2" +fi + +if [ "$1" = "stopInstance" ]; then + if [ "$2" = "" ]; then + echo "Enter Instance Name" + exit 1 + fi + instanceName="$2" +fi + +if [ "$1" = "startInstance" ]; then + if [ "$2" = "" ]; then + echo "Enter Instance Name" + exit 1 + fi + instanceName="$2" +fi + +if [ "$1" = "assignElasticIP" ]; then + if [ "$2" = "" ]; then + echo "Enter Instance Name" + exit 1 + else + instanceName="$2" + fi + if [ "$3" = "" ]; then + echo "Enter Elastic IP Address" + exit 1 + else + IPAddress="$3" + fi +fi + +if [ "$1" = "deassignElasticIP" ]; then + if [ "$2" = "" ]; then + echo "Enter Elastic IP Address" + exit 1 + else + IPAddress="$2" + fi +fi + + +test -f /usr/local/mariadb/columnstore/post/functions && . /usr/local/mariadb/columnstore/post/functions + +ec2=`$prefix/mariadb/columnstore/bin/getConfig Installation EC2_HOME` + +if [ $ec2 == "unassigned" ]; then + if [ "$1" = "getPrivateIP" ]; then + echo "stopped" + exit 1 + else + echo "unknown" + exit 1 + fi +fi + +java=`$prefix/mariadb/columnstore/bin/getConfig Installation JAVA_HOME` +path=`$prefix/mariadb/columnstore/bin/getConfig Installation EC2_PATH` + +export PATH=$path +export EC2_HOME=$ec2 +export JAVA_HOME=$java + +# get Keys and region +AmazonAccessKeyFile=`$prefix/mariadb/columnstore/bin/getConfig Installation AmazonAccessKey` +if [ $AmazonAccessKeyFile == "unassigned" ]; then + echo "FAILED: missing Config Setting AmazonAccessKey : $AmazonAccessKeyfile" + exit 1 +fi + +AmazonSecretKeyFile=`$prefix/mariadb/columnstore/bin/getConfig Installation AmazonSecretKey` +if [ $AmazonSecretKeyFile == "unassigned" ]; then + echo "FAILED: missing Config Setting AmazonSecretKeyFile : $AmazonSecretKeyFile" + exit 1 +fi + +AmazonAccessKey=`cat $AmazonAccessKeyFile` +AmazonSecretKey=`cat $AmazonSecretKeyFile` + +Region=`$prefix/mariadb/columnstore/bin/getConfig Installation AmazonRegion` +subnet=`$prefix/mariadb/columnstore/bin/getConfig Installation AmazonSubNetID` + +if test ! -f $AmazonAccessKeyfile ; then + echo "FAILED: missing AmazonAccessKeyfile : $AmazonAccessKeyfile" + exit 1 +fi + +if test ! -f $AmazonSecretKeyfile ; then + echo "FAILED: missing AmazonSecretKeyfile : $AmazonSecretKeyfile" + exit 1 +fi + +#default instance to null +instance="" + +describeInstanceFile="/tmp/describeInstance.txt" +touch $describeInstanceFile + +describeInstance() { + ec2-describe-instances -O $AmazonAccessKey -W $AmazonSecretKey --region $Region > $describeInstanceFile 2>&1 +} + +#call at start up +describeInstance + +getInstance() { + if [ "$instance" != "" ]; then + echo $instance + return + fi + + # first get local IP Address + localIP=`ifconfig eth0 | grep -m 1 "inet" | awk '{print substr($2,0,20)}'` + + #get local Instance ID + instance=`cat $describeInstanceFile | grep -m 1 -w $localIP | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'` + if [ "$instance" == "" ]; then + describeInstance + fi + instance=`cat $describeInstanceFile | grep -m 1 -w $localIP | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'` + + echo $instance + return +} + +getInstancePrivate() { + if [ "$instance" != "" ]; then + echo $instance + return + fi + + # first get local IP Address + localIP=`ifconfig eth0 | grep -m 1 "inet" | awk '{print substr($2,0,20)}'` + + #get local Instance ID + instance=`cat $describeInstanceFile | grep -m 1 -w $localIP | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'` + if [ "$instance" == "" ]; then + describeInstance + fi + instance=`cat $describeInstanceFile | grep -m 1 -w $localIP | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'` + + return +} + + +getZone() { + #get from Columnstore.xml if it's there, if not, get from instance then store + zone=`$prefix/mariadb/columnstore/bin/getConfig Installation AmazonZone` + + if [ "$zone" = "unassigned" ] || [ "$zone" = "" ]; then + #get local Instance ID + getInstancePrivate >/dev/null 2>&1 + #get zone + if [ "$subnet" == "unassigned" ]; then + zone=`cat $describeInstanceFile | grep -m 1 $instance | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $11}'` + if [ "$zone" == "" ]; then + describeInstance + fi + zone=`cat $describeInstanceFile | grep -m 1 $instance | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $11}'` + + else + zone=`cat $describeInstanceFile | grep -m 1 $instance | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $11}'` + if [ "$zone" == "" ]; then + describeInstance + fi + zone=`cat $describeInstanceFile | grep -m 1 $instance | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $11}'` + fi + $prefix/mariadb/columnstore/bin/setConfig Installation AmazonZone $zone + fi + + echo $zone + return +} + +getPrivateIP() { + #get instance info + grep -B1 -A4 -m 1 $instanceName $describeInstanceFile > /tmp/instanceInfo_$instanceName 2>&1 + if [ `cat /tmp/instanceInfo_$instanceName | wc -c` -eq 0 ]; then + describeInstance + fi + grep -B1 -A4 -m 1 $instanceName $describeInstanceFile > /tmp/instanceInfo_$instanceName 2>&1 + + #check if running or terminated + cat /tmp/instanceInfo_$instanceName | grep running > /tmp/instanceStatus_$instanceName + if [ `cat /tmp/instanceStatus_$instanceName | wc -c` -eq 0 ]; then + # not running + cat /tmp/instanceInfo_$instanceName | grep pending > /tmp/instanceStatus_$instanceName + if [ `cat /tmp/instanceStatus_$instanceName | wc -c` -ne 0 ]; then + describeInstance + echo "stopped" + exit 1 + else + cat /tmp/instanceInfo_$instanceName | grep terminated > /tmp/instanceStatus_$instanceName + if [ `cat /tmp/instanceStatus_$instanceName | wc -c` -ne 0 ]; then + echo "terminated" + exit 1 + else + cat /tmp/instanceInfo_$instanceName | grep shutting-down > /tmp/instanceStatus_$instanceName + if [ `cat /tmp/instanceStatus_$instanceName | wc -c` -ne 0 ]; then + echo "terminated" + exit 1 + else + echo "stopped" + exit 1 + fi + fi + fi + fi + + #running, get priviate IP Address + if [ "$subnet" == "unassigned" ]; then + IpAddr=`head -n 2 /tmp/instanceInfo_$instanceName | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $14}'` + else + IpAddr=`head -n 2 /tmp/instanceInfo_$instanceName | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $14}'` + fi + + echo $IpAddr + exit 0 +} + +getType() { + #get local Instance ID + getInstancePrivate >/dev/null 2>&1 + #get Type + if [ "$subnet" == "unassigned" ]; then + instanceType=`cat $describeInstanceFile | grep -m 1 $instance | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $9}'` + if [ "$instanceType" == "" ]; then + describeInstance + fi + instanceType=`cat $describeInstanceFile | grep -m 1 $instance | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $9}'` + + else + instanceType=`cat $describeInstanceFile | grep -m 1 $instance | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $9}'` + if [ "$instanceType" == "" ]; then + describeInstance + fi + instanceType=`cat $describeInstanceFile | grep -m 1 $instance | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $9}'` + + fi + + echo $instanceType + return +} + +getKey() { + #get local Instance ID + getInstancePrivate >/dev/null 2>&1 + #get Key + if [ "$subnet" == "unassigned" ]; then + key=`cat $describeInstanceFile | grep -m 1 $instance | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $7}'` + if [ "$key" == "" ]; then + describeInstance + fi + key=`cat $describeInstanceFile | grep -m 1 $instance | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $7}'` + + else + key=`cat $describeInstanceFile | grep -m 1 $instance | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $7}'` + if [ "$key" == "" ]; then + describeInstance + fi + key=`cat $describeInstanceFile | grep -m 1 $instance | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $7}'` + + fi + + echo $key + return +} + +getAMI() { + #get local Instance ID + getInstancePrivate >/dev/null 2>&1 + #get AMI + ami=`cat $describeInstanceFile | grep -m 1 $instance | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $3}'` + if [ "$ami" == "" ]; then + describeInstance + fi + ami=`cat $describeInstanceFile | grep -m 1 $instance | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $3}'` + + echo $ami + return +} + +getGroup() { + #get local Instance ID + getInstancePrivate >/dev/null 2>&1 + #get group + if [ "$subnet" == "unassigned" ]; then + group=`grep -B1 -A4 -m 1 $instance $describeInstanceFile | grep -m 1 RESERVATION | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $4}'` + if [ "$group" == "" ]; then + describeInstance + fi + group=`grep -B1 -A4 -m 1 $instance $describeInstanceFile | grep -m 1 RESERVATION | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $4}'` + if [ "$group" == "" ]; then + group=`grep -B1 -A4 -m 1 $instance $describeInstanceFile | grep -m 1 INSTANCE | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $21}'` + fi + else + group=`grep -B1 -A6 -m 1 $instance $describeInstanceFile | grep -m 1 GROUP | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'` + if [ "$group" == "" ]; then + describeInstance + fi + group=`grep -B1 -A6 -m 1 $instance $describeInstanceFile | grep -m 1 GROUP | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'` + if [ "$group" == "" ]; then + group=`grep -B1 -A4 -m 1 $instance $describeInstanceFile | grep -m 1 INSTANCE | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $21}'` + fi + fi + + echo $group + return +} + +getProfile() { + #get local Instance ID + getInstancePrivate >/dev/null 2>&1 + #get Type + if [ "$subnet" == "unassigned" ]; then + instanceProfile=`cat $describeInstanceFile | grep -m 1 $instance | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $23}'` + if [ "$instanceProfile" == "" ]; then + describeInstance + fi + instanceProfile=`cat $describeInstanceFile | grep -m 1 $instance | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $23}'` + + else + instanceProfile=`cat $describeInstanceFile | grep -m 1 $instance | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $22}'` + if [ "$instanceProfile" == "" ]; then + describeInstance + fi + instanceProfile=`cat $describeInstanceFile | grep -m 1 $instance | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $22}'` + + fi + + echo $instanceProfile + return +} + +launchInstance() { + #get publickey + getKey >/dev/null 2>&1 + if [ "$group" = "unassigned" ]; then + #get group + getGroup >/dev/null 2>&1 + fi + #get AMI + getAMI >/dev/null 2>&1 + #get Zone + getZone >/dev/null 2>&1 + if [ "$instanceType" = "unassigned" ]; then + #get type + getType >/dev/null 2>&1 + fi + #get AMI Profile + getProfile >/dev/null 2>&1 + + if [ "$subnet" == "unassigned" ]; then + #NOT VPC + if [ "$instanceProfile" = "" ] || [ "$instanceProfile" = "default" ]; then + newInstance=`ec2-run-instances -O $AmazonAccessKey -W $AmazonSecretKey -k $key -g $group -t $instanceType -z $zone --region $Region $ami | grep -m 1 INSTANCE | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'` + else + newInstance=`ec2-run-instances -O $AmazonAccessKey -W $AmazonSecretKey -k $key -g $group -t $instanceType -z $zone -p $instanceProfile --region $Region $ami | grep -m 1 INSTANCE | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'` + fi + else # VPC + if [ "$instanceProfile" = "" ] || [ "$instanceProfile" = "default" ]; then + if [ "$group" != "default" ]; then + if [ "$IPaddress" = "autoassign" ]; then + newInstance=`ec2-run-instances -O $AmazonAccessKey -W $AmazonSecretKey -k $key -g $group -t $instanceType -z $zone --region $Region -s $subnet $ami | grep -m 1 INSTANCE | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'` + else + newInstance=`ec2-run-instances -O $AmazonAccessKey -W $AmazonSecretKey -k $key -g $group -t $instanceType -z $zone --region $Region -s $subnet --private-ip-address $IPaddress $ami | grep -m 1 INSTANCE | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'` + fi + else + if [ "$IPaddress" = "autoassign" ]; then + newInstance=`ec2-run-instances -O $AmazonAccessKey -W $AmazonSecretKey -k $key -t $instanceType -z $zone --region $Region -s $subnet $ami | grep -m 1 INSTANCE | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'` + else + newInstance=`ec2-run-instances --O $AmazonAccessKey -W $AmazonSecretKey -k $key -t $instanceType -z $zone --region $Region -s $subnet --private-ip-address $IPaddress $ami | grep -m 1 INSTANCE | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'` + fi + fi + else + if [ "$group" != "default" ]; then + if [ "$IPaddress" = "autoassign" ]; then + newInstance=`ec2-run-instances -O $AmazonAccessKey -W $AmazonSecretKey -k $key -g $group -t $instanceType -z $zone -p $instanceProfile --region $Region -s $subnet $ami | grep -m 1 INSTANCE | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'` + else + newInstance=`ec2-run-instances -O $AmazonAccessKey -W $AmazonSecretKey -k $key -g $group -t $instanceType -z $zone -p $instanceProfile --region $Region -s $subnet --private-ip-address $IPaddress $ami | grep -m 1 INSTANCE | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'` + fi + else + if [ "$IPaddress" = "autoassign" ]; then + newInstance=`ec2-run-instances -O $AmazonAccessKey -W $AmazonSecretKey -k $key -t $instanceType -z $zone -p $instanceProfile --region $Region -s $subnet $ami | grep -m 1 INSTANCE | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'` + else + newInstance=`ec2-run-instances -O $AmazonAccessKey -W $AmazonSecretKey -k $key -t $instanceType -z $zone -p $instanceProfile --region $Region -s $subnet --private-ip-address $IPaddress $ami | grep -m 1 INSTANCE | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'` + fi + fi + fi + fi + echo $newInstance + return +} + +terminateInstance() { + #terminate Instance + ec2-terminate-instances -O $AmazonAccessKey -W $AmazonSecretKey --region $Region $instanceName > /tmp/termInstanceInfo_$instanceName 2>&1 + return +} + +stopInstance() { + #terminate Instance + ec2-stop-instances -O $AmazonAccessKey -W $AmazonSecretKey --region $Region $instanceName > /tmp/stopInstanceInfo_$instanceName 2>&1 + return +} + +startInstance() { + #terminate Instance + ec2-start-instances -O $AmazonAccessKey -W $AmazonSecretKey --region $Region $instanceName > /tmp/startInstanceInfo_$instanceName 2>&1 + + cat /tmp/startInstanceInfo_$instanceName | grep INSTANCE > /tmp/startInstanceStatus_$instanceName + if [ `cat /tmp/startInstanceStatus_$instanceName | wc -c` -eq 0 ]; then + echo "Failed, check /tmp/startInstanceInfo_$instanceName" + exit 1 + fi + echo "Success" + exit 0 +} + +assignElasticIP() { + #terminate Instance + + if [ "$subnet" == "unassigned" ]; then + ec2-associate-address -O $AmazonAccessKey -W $AmazonSecretKey -i $instanceName $IPAddress > /tmp/assignElasticIPInfo_$IPAddress 2>&1 + else + EIP=`ec2-describe-addresses -O $AmazonAccessKey -W $AmazonSecretKey --region $Region $IPAddress | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $4}'` + ec2-associate-address -O $AmazonAccessKey -W $AmazonSecretKey --region $Region -i $instanceName -a $EIP > /tmp/assignElasticIPInfo_$IPAddress 2>&1 + fi + + cat /tmp/assignElasticIPInfo_$IPAddress | grep ADDRESS > /tmp/assignElasticIPStatus_$IPAddress + if [ `cat /tmp/assignElasticIPStatus_$IPAddress | wc -c` -eq 0 ]; then + echo "Failed, check /tmp/assignElasticIPInfo_$IPAddress" + exit 1 + fi + + echo "Success" + exit 0 +} + +deassignElasticIP() { + #terminate Instance + ec2-disassociate-address -O $AmazonAccessKey -W $AmazonSecretKey $IPAddress > /tmp/deassignElasticIPInfo_$IPAddress 2>&1 + + cat /tmp/deassignElasticIPInfo_$IPAddress | grep ADDRESS > /tmp/deassignElasticIPStatus_$IPAddress + if [ `cat /tmp/deassignElasticIPStatus_$IPAddress | wc -c` -eq 0 ]; then + echo "Failed, check /tmp/deassignElasticIPStatus_$IPAddress" + exit 1 + fi + + echo "Success" + exit 0 +} + +getSubnet() { + #get local Instance ID + getInstancePrivate >/dev/null 2>&1 + #get Subnet + subnet=`cat $describeInstanceFile | grep -m 1 $instance | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $16}'` + if [ "$subnet" == "" ]; then + describeInstance + fi + subnet=`cat $describeInstanceFile | grep -m 1 $instance | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $16}'` + + if [[ $subnet == *"subnet"* ]] + then + echo $subnet + else + echo "failed" + fi + + return +} + + +case "$1" in + getInstance) + getInstance + ;; + getZone) + getZone + ;; + getPrivateIP) + getPrivateIP + ;; + getKey) + getKey + ;; + getAMI) + getAMI + ;; + getType) + getType + ;; + launchInstance) + launchInstance + ;; + terminateInstance) + terminateInstance + ;; + stopInstance) + stopInstance + ;; + startInstance) + startInstance + ;; + assignElasticIP) + assignElasticIP + ;; + deassignElasticIP) + deassignElasticIP + ;; + getProfile) + getProfile + ;; + getGroup) + getGroup + ;; + getSubnet) + getSubnet + ;; + *) + echo $"Usage: $0 {launchInstance|getInstance|getZone|getPrivateIP|getType|getKey|getAMI|terminateInstance|startInstance|assignElasticIP|deassignElasticIP|getProfile|stopInstance|getGroup|getSubnet}" + exit 1 +esac + +exit $? diff --git a/oam/cloud/MCSVolumeCmds.sh b/oam/cloud/MCSVolumeCmds.sh new file mode 100644 index 000000000..7bac1a7dc --- /dev/null +++ b/oam/cloud/MCSVolumeCmds.sh @@ -0,0 +1,337 @@ +#! /bin/sh +# MCSVolumeCmds.sh +# describe, detach and attach Volume Storage from a Cloud environment +# +# 1. Amazon EC2 + +prefix=/usr/local + +#check command +if [ "$1" = "" ]; then + echo "Enter Command Name: {create|describe|detach|attach|delete|createTag}" + exit 1 +fi + +if [ "$1" = "create" ]; then + if [ "$2" = "" ]; then + echo "Enter size of the volume, in GiB (1-1024)" + exit 1 + fi + volumeSize="$2" + + #get module-type + if [ "$3" = "" ]; then + echo "Enter Module Type" + exit 1 + fi + moduleType="$3" +fi + +if [ "$1" = "describe" ]; then + if [ "$2" = "" ]; then + echo "Enter Volume Name" + exit 1 + fi + volumeName="$2" +fi + +if [ "$1" = "detach" ]; then + if [ "$2" = "" ]; then + echo "Enter Volume Name" + exit 1 + fi + volumeName="$2" +fi + +if [ "$1" = "attach" ]; then + if [ "$2" = "" ]; then + echo "Enter Volume Name" + exit 1 + fi + volumeName="$2" + + #get instance-name and device-name + if [ "$3" = "" ]; then + echo "Enter Instance Name" + exit 1 + fi + instanceName="$3" + + if [ "$4" = "" ]; then + echo "Enter Device Name" + exit 1 + fi + deviceName="$4" +fi + +if [ "$1" = "delete" ]; then + if [ "$2" = "" ]; then + echo "Enter Volume Name" + exit 1 + fi + volumeName="$2" +fi + +if [ "$1" = "createTag" ]; then + if [ "$2" = "" ]; then + echo "Enter Resource Name" + exit 1 + fi + resourceName="$2" + + if [ "$3" = "" ]; then + echo "Enter Tag Name" + exit 1 + fi + tagName="$3" + + if [ "$4" = "" ]; then + echo "Enter Tag Value" + exit 1 + fi + tagValue="$4" +fi + + +test -f $prefix/mariadb/columnstore/post/functions && . $prefix/mariadb/columnstore/post/functions + +ec2=`$prefix/mariadb/columnstore/bin/getConfig Installation EC2_HOME` + +if [ $ec2 == "unassigned" ]; then + STATUS="unknown" + RETVAL=1 +fi + +java=`$prefix/mariadb/columnstore/bin/getConfig Installation JAVA_HOME` +path=`$prefix/mariadb/columnstore/bin/getConfig Installation EC2_PATH` + +export PATH=$path +export EC2_HOME=$ec2 +export JAVA_HOME=$java + +# get Keys and region +AmazonAccessKeyFile=`$prefix/mariadb/columnstore/bin/getConfig Installation AmazonAccessKey` +if [ $AmazonAccessKeyFile == "unassigned" ]; then + echo "FAILED: missing Config Setting AmazonAccessKey : $AmazonAccessKeyfile" + exit 1 +fi + +AmazonSecretKeyFile=`$prefix/mariadb/columnstore/bin/getConfig Installation AmazonSecretKey` +if [ $AmazonSecretKeyFile == "unassigned" ]; then + echo "FAILED: missing Config Setting AmazonSecretKeyFile : $AmazonSecretKeyFile" + exit 1 +fi + +AmazonAccessKey=`cat $AmazonAccessKeyFile` +AmazonSecretKey=`cat $AmazonSecretKeyFile` + +if test ! -f $AmazonAccessKeyfile ; then + echo "FAILED: missing AmazonAccessKeyfile : $AmazonAccessKeyfile" + exit 1 +fi + +if test ! -f $AmazonSecretKeyfile ; then + echo "FAILED: missing AmazonSecretKeyfile : $AmazonSecretKeyfile" + exit 1 +fi + +Region=`$prefix/mariadb/columnstore/bin/getConfig Installation AmazonRegion` + + +checkInfostatus() { + #check if attached + cat /tmp/volumeInfo_$volumeName | grep attached > /tmp/volumeStatus_$volumeName + if [ `cat /tmp/volumeStatus_$volumeName | wc -c` -ne 0 ]; then + STATUS="attached" + RETVAL=0 + return + fi + #check if available + cat /tmp/volumeInfo_$volumeName | grep available > /tmp/volumeStatus_$volumeName + if [ `cat /tmp/volumeStatus_$volumeName | wc -c` -ne 0 ]; then + STATUS="available" + RETVAL=0 + return + fi + #check if detaching + cat /tmp/volumeInfo_$volumeName | grep detaching > /tmp/volumeStatus_$volumeName + if [ `cat /tmp/volumeStatus_$volumeName | wc -c` -ne 0 ]; then + STATUS="detaching" + RETVAL=0 + return + fi + #check if attaching + cat /tmp/volumeInfo_$volumeName | grep attaching > /tmp/volumeStatus_$volumeName + if [ `cat /tmp/volumeStatus_$volumeName | wc -c` -ne 0 ]; then + STATUS="attaching" + RETVAL=0 + return + fi + #check if doesn't exist + cat /tmp/volumeInfo_$volumeName | grep "does not exist" > /tmp/volumeStatus_$volumeName + if [ `cat /tmp/volumeStatus_$volumeName | wc -c` -ne 0 ]; then + STATUS="does-not-exist" + RETVAL=1 + return + fi + #check if reports already attached from attach command + cat /tmp/volumeInfo_$volumeName | grep "already attached" > /tmp/volumeStatus_$volumeName + if [ `cat /tmp/volumeStatus_$volumeName | wc -c` -ne 0 ]; then + STATUS="already-attached" + RETVAL=1 + return + fi + #any other, unknown error + STATUS="unknown" + RETVAL=1 + return +} + +createvolume() { + # get zone + zone=`$prefix/mariadb/columnstore/bin/MCSInstanceCmds.sh getZone` + + if [ $moduleType == "um" ]; then + # get type + volumeType=`$prefix/mariadb/columnstore/bin/getConfig Installation UMVolumeType` + if [ $volumeType == "io1" ]; then + # get IOPS + volumeIOPS=`$prefix/mariadb/columnstore/bin/getConfig Installation UMVolumeIOPS` + fi + else # pm + # get type + volumeType=`$prefix/mariadb/columnstore/bin/getConfig Installation PMVolumeType` + if [ $volumeType == "io1" ]; then + # get IOPS + volumeIOPS=`$prefix/mariadb/columnstore/bin/getConfig Installation PMVolumeIOPS` + fi + fi + + #create volume + if [ $volumeType == "io1" ]; then + volume=`ec2-create-volume -O $AmazonAccessKey -W $AmazonSecretKey --region $Region -z $zone -s $volumeSize -t $volumeType -iops $volumeIOPS | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'` + else + volume=`ec2-create-volume -O $AmazonAccessKey -W $AmazonSecretKey --region $Region -z $zone -s $volumeSize -t $volumeType | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'` + fi + + echo $volume + return +} + +describevolume() { + #describe volume + ec2-describe-volumes -O $AmazonAccessKey -W $AmazonSecretKey --region $Region $volumeName > /tmp/volumeInfo_$volumeName 2>&1 + + checkInfostatus + echo $STATUS + return +} + +detachvolume() { + #detach volume + ec2-detach-volume -O $AmazonAccessKey -W $AmazonSecretKey --region $Region $volumeName > /tmp/volumeInfo_$volumeName 2>&1 + + checkInfostatus + if [ $STATUS == "detaching" ]; then + retries=1 + while [ $retries -ne 60 ]; do + #retry until it's attached + ec2-detach-volume -O $AmazonAccessKey -W $AmazonSecretKey --region $Region $volumeName > /tmp/volumeInfo_$volumeName 2>&1 + + checkInfostatus + if [ $STATUS == "available" ]; then + echo "available" + exit 0 + fi + ((retries++)) + sleep 1 + done + test -f /usr/local/mariadb/columnstore/post/functions && . /usr/local/mariadb/columnstore/post/functions + $prefix/mariadb/columnstore/bin/cplogger -w 100 "detachvolume failed: $STATUS" + echo "failed" + exit 1 + fi + + if [ $STATUS == "available" ]; then + echo "available" + exit 0 + fi + + test -f /usr/local/mariadb/columnstore/post/functions && . /usr/local/mariadb/columnstore/post/functions + $prefix/mariadb/columnstore/bin/cplogger -w 100 "detachvolume failed status: $STATUS" + echo $STATUS + exit 1 +} + +attachvolume() { + + #detach volume + ec2-attach-volume -O $AmazonAccessKey -W $AmazonSecretKey --region $Region $volumeName -i $instanceName -d $deviceName > /tmp/volumeInfo_$volumeName 2>&1 + + checkInfostatus + if [ $STATUS == "attaching" -o $STATUS == "already-attached" ]; then + retries=1 + while [ $retries -ne 60 ]; do + #check status until it's attached + describevolume + if [ $STATUS == "attached" ]; then + echo "attached" + exit 0 + fi + ((retries++)) + sleep 1 + done + test -f /usr/local/mariadb/columnstore/post/functions && . /usr/local/mariadb/columnstore/post/functions + $prefix/mariadb/columnstore/bin/cplogger -w 100 "attachvolume failed: $STATUS" + echo "failed" + exit 1 + fi + + if [ $STATUS == "attached" ]; then + echo "attached" + exit 0 + fi + + test -f /usr/local/mariadb/columnstore/post/functions && . /usr/local/mariadb/columnstore/post/functions + $prefix/mariadb/columnstore/bin/cplogger -w 100 "attachvolume failed: $STATUS" + echo $STATUS + exit 1 +} + +deletevolume() { + #delete volume + ec2-delete-volume -O $AmazonAccessKey -W $AmazonSecretKey --region $Region $volumeName > /tmp/deletevolume_$volumeName 2>&1 + return +} + +createTag() { + #create tag + ec2-create-tags -O $AmazonAccessKey -W $AmazonSecretKey --region $Region $resourceName --tag $tagName=$tagValue > /tmp/createTag_$volumeName 2>&1 + return +} + +case "$1" in + create) + createvolume + ;; + describe) + describevolume + ;; + detach) + detachvolume + ;; + attach) + attachvolume + ;; + delete) + deletevolume + ;; + createTag) + createTag + ;; + *) + echo $"Usage: $0 {create|describe|detach|attach|delete|}" + exit 1 +esac + +exit $? diff --git a/oam/etc/ConsoleCmds.xml b/oam/etc/ConsoleCmds.xml index 37c7d14f9..88d98ffed 100644 --- a/oam/etc/ConsoleCmds.xml +++ b/oam/etc/ConsoleCmds.xml @@ -34,7 +34,12 @@ STATUS to to view statistics and progress - AVAILABLE + findObjectFile + Get the name of the directory containing the first file of the object + Object ID (OID) of object + Schema + Table + Column getModuleTypeConfig diff --git a/oam/install_scripts/binary_installer.sh b/oam/install_scripts/binary_installer.sh index ea6123f13..efd202cf1 100644 --- a/oam/install_scripts/binary_installer.sh +++ b/oam/install_scripts/binary_installer.sh @@ -234,7 +234,7 @@ if { $INSTALLTYPE == "initial"} { send_user "Run Module Installer " send " \n" send date\n - send "ssh $USERNAME@$SERVER '$INSTALLDIR/bin/module_installer.sh --module=$MODULETYPE --port=$MYSQLPORT --installdir=$INSTALLDIR $MODULETYPE'\n" + send "ssh $USERNAME@$SERVER '$INSTALLDIR/bin/module_installer.sh --module=$MODULETYPE --port=$MYSQLPORT --installdir=$INSTALLDIR'\n" set timeout 10 expect { "word: " { send "$PASSWORD\n" } diff --git a/oam/install_scripts/columnstore b/oam/install_scripts/columnstore index c346e3eb0..315256cd9 100644 --- a/oam/install_scripts/columnstore +++ b/oam/install_scripts/columnstore @@ -57,8 +57,10 @@ if [ "x$has_um" = x ]; then fi user=$USER +sudo=sudo if [ -z "$user" ]; then user=root + sudo=" " fi checkInstallSetup() { @@ -66,7 +68,7 @@ checkInstallSetup() { if [ $InitialInstallFlag != "y" ]; then echo "Please run the postConfigure install script, check the Installation Guide" echo "for additional details" - rm -f /var/lock/subsys/columnstore + sudo rm -f /var/lock/subsys/columnstore exit 1 fi } @@ -79,13 +81,13 @@ start() { exit 0 fi - (mkdir -p /var/lock/subsys && touch /var/lock/subsys/columnstore) >/dev/null 2>&1 + (sudo mkdir -p /var/lock/subsys && touch /var/lock/subsys/columnstore) >/dev/null 2>&1 if [ -x $InstallDir/bin/columnstore.pre-start ]; then $InstallDir/bin/columnstore.pre-start if [ $? -ne 0 ]; then echo "Error running MariaDB Columnstore pre-start script, not starting MariaDB Columnstore" - rm -f /var/lock/subsys/columnstore + sudo rm -f /var/lock/subsys/columnstore exit 1 fi fi @@ -106,21 +108,21 @@ start() { RETVAL=0 echo "Starting MariaDB Columnstore Database Platform" - rm -f /tmp/StopColumnstore + sudo rm -f /tmp/StopColumnstore exec $InstallDir/bin/run.sh $InstallDir/bin/ProcMon > /dev/null 2>&1 & return $RETVAL } stop() { echo "Shutting down MariaDB Columnstore Database Platform" - touch /tmp/StopColumnstore - pkill ProcMon - pkill ProcMgr + sudo touch /tmp/StopColumnstore + sudo pkill ProcMon + sudo pkill ProcMgr sleep 1 $InstallDir/bin/clearShm RETVAL=$? - rm -f /var/lock/subsys/columnstore - fuser -k 8604/tcp > /dev/null 2>&1 + sudo rm -f /var/lock/subsys/columnstore + sudo fuser -k 8604/tcp > /dev/null 2>&1 test -f $InstallDir/mysql/mysql-Columnstore || return $RETVAL $InstallDir/mysql/mysql-Columnstore stop > /dev/null 2>&1 if [ -x $InstallDir/bin/columnstore.post-stop ]; then diff --git a/oam/install_scripts/columnstoreAlias b/oam/install_scripts/columnstoreAlias index 2cf611819..b1dc3dd9c 100644 --- a/oam/install_scripts/columnstoreAlias +++ b/oam/install_scripts/columnstoreAlias @@ -3,6 +3,7 @@ alias mcsmysql='/usr/local/mariadb/columnstore/mysql/bin/mysql --defaults-file=/usr/local/mariadb/columnstore/mysql/my.cnf -u root' alias ma=/usr/local/mariadb/columnstore/bin/mcsadmin alias mcsadmin=/usr/local/mariadb/columnstore/bin/mcsadmin +alias cpimport=/usr/local/mariadb/columnstore/bin/cpimport alias home='cd /usr/local/mariadb/columnstore' alias log='cd /var/log/mariadb/columnstore/' alias core='cd /var/log/mariadb/columnstore/corefiles' diff --git a/oam/install_scripts/columnstoreSyslog7 b/oam/install_scripts/columnstoreSyslog7 index 87bababaa..0ed37f975 100644 --- a/oam/install_scripts/columnstoreSyslog7 +++ b/oam/install_scripts/columnstoreSyslog7 @@ -1,4 +1,4 @@ -# MariaDb Columnstore Database Platform Logging +# MariaDB Columnstore Database Platform Logging local1.crit -/var/log/mariadb/columnstore/crit.log local1.err -/var/log/mariadb/columnstore/err.log local1.warning -/var/log/mariadb/columnstore/warning.log diff --git a/oam/install_scripts/module_installer.sh b/oam/install_scripts/module_installer.sh index 5b0fb2e01..19d76bc6f 100755 --- a/oam/install_scripts/module_installer.sh +++ b/oam/install_scripts/module_installer.sh @@ -113,18 +113,6 @@ if [ $EUID -eq 0 -a -f $COLUMNSTORE_INSTALL_DIR/local/rc.local.columnstore ]; th fi fi -if [ $user != "root" ]; then - echo "Setup .bashrc on Module for non-root" - - eval userhome=~$user - bashFile=$userhome/.bashrc - touch ${bashFile} - - echo " " >> ${bashFile} - echo "export COLUMNSTORE_INSTALL_DIR=$COLUMNSTORE_INSTALL_DIR" >> ${bashFile} - echo "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$COLUMNSTORE_INSTALL_DIR/lib:$COLUMNSTORE_INSTALL_DIR/mysql/lib/mysql" >> ${bashFile} -fi - plugin=`$COLUMNSTORE_INSTALL_DIR/bin/getConfig SystemConfig DataFilePlugin` if [ -n "$plugin" ]; then echo "Setup .bashrc on Module for local-query" diff --git a/oam/install_scripts/post-install b/oam/install_scripts/post-install index 38ad26fe6..e24b0dfb5 100755 --- a/oam/install_scripts/post-install +++ b/oam/install_scripts/post-install @@ -8,9 +8,12 @@ prefix=/usr/local installdir=$prefix/mariadb/columnstore rpmmode=install user=$USER +sudo=sudo if [ -z "$user" ]; then - user=root + user=root + sudo=" " fi + quiet=0 for arg in "$@"; do @@ -81,27 +84,52 @@ fi # softlink for libperl.sp, used by cplogger sudo ln -s /usr/lib64/perl5/CORE/libperl.so /usr/lib64/libperl.so >/dev/null 2>&1 -# TODO-based on early experience with HDP it doesn't install bigtop -# so the code below will not correctly detect JAVA_HOME. Need to -# decide how to handle -# Autodetect JAVA_HOME if not defined -#if [ -e /usr/libexec/bigtop-detect-javahome ]; then -#. /usr/libexec/bigtop-detect-javahome -#elif [ -e /usr/lib/bigtop-utils/bigtop-detect-javahome ]; then -#. /usr/lib/bigtop-utils/bigtop-detect-javahome -#fi -#ln -sf $JAVA_HOME/jre/lib/amd64/server/libjvm.so -#chown -h $user.$user libjvm.so >/dev/null 2>&1 +#setup .bashrc +if [ $user = "root" ]; then + bashFile=/root/.bashrc +else + bashFile=$HOME/.bashrc +fi + +touch ${bashFile} +/bin/cp -f ${bashFile} ${bashFile}.columnstoreSave > /dev/null 2>&1 + +if [ $user != "root" ]; then + + egrep -qs 'MariaDB Columnstore Non-Root' ${bashFile} + rc1=$? + if [ $rc1 -ne 0 ]; then + echo " " >> ${bashFile} + echo "# MariaDB Columnstore Non-Root Environment Variables" >> ${bashFile} + echo "export COLUMNSTORE_INSTALL_DIR=$COLUMNSTORE_INSTALL_DIR" >> ${bashFile} + echo "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$COLUMNSTORE_INSTALL_DIR/lib:$COLUMNSTORE_INSTALL_DIR/mysql/lib/mysql" >> ${bashFile} + . ${bashFile} + fi +fi + +if [ $installdir != "/usr/local/mariadb/columnstore" ]; then + sed -i -e s@/usr/local/mariadb/columnstore@$installdir@g $installdir/bin/columnstoreAlias +fi +egrep -qs 'MariaDB Columnstore Database Platform Alias Commands' ${bashFile} +rc1=$? +egrep -qs 'MariaDB Columnstore Alias Commands' ${bashFile} +rc2=$? +if [ $rc1 -ne 0 -a $rc2 -ne 0 ]; then + cat $installdir/bin/columnstoreAlias >> ${bashFile} + echo " " >> ${bashFile} +fi cd / -test -d /var/log/mariadb/columnstore || mkdir /var/log/mariadb/columnstore >/dev/null 2>&1 +test -d /var/log/mariadb || sudo mkdir /var/log/mariadb >/dev/null 2>&1 +test -d /var/log/mariadb/columnstore || sudo mkdir /var/log/mariadb/columnstore >/dev/null 2>&1 +sudo chmod -R 777 /var/log/mariadb + test -d /var/log/mariadb/columnstore/archive || mkdir /var/log/mariadb/columnstore/archive >/dev/null 2>&1 test -d /var/log/mariadb/columnstore/corefiles || mkdir /var/log/mariadb/columnstore/corefiles >/dev/null 2>&1 test -d /var/log/mariadb/columnstore/trace || mkdir /var/log/mariadb/columnstore/trace >/dev/null 2>&1 test -h /var/log/mariadb/columnstore/data && rm -f /var/log/mariadb/columnstore/data # make sure trace dir is world-writable and sticky -chmod 1755 /var/log/mariadb/columnstore/trace >/dev/null 2>&1 test -d $installdir/data || mkdir $installdir/data test -d $installdir/data1 || mkdir $installdir/data1 test -d $installdir/data1/systemFiles || mkdir $installdir/data1/systemFiles @@ -180,16 +208,20 @@ if [ $user = "root" ]; then chmod 777 -R /dev/shm rm -f /etc/default/columnstore else + sudo chmod 777 /tmp + sudo rm -fr /tmp/* > /dev/null 2>&1 sudo $installdir/bin/syslogSetup.sh --installdir=$installdir install > /tmp/syslog_install.log 2>&1 sudo chown $user:$user $installdir/etc/Columnstore.xml - sudo chmod 755 /var/log - sudo chmod -R 755 /var/log/mariadb/columnstore sudo chmod -R 777 /dev/shm + sudo chmod 777 /var/lock/subsys sudo rm -f /var/lock/subsys/mysql-Columnstore - sudo rm -f /tmp/StopColumnstore + sudo chmod 777 /etc/fstab sed -i -e s@/usr/local/mariadb/columnstore@$installdir@g $installdir/bin/columnstore.def sudo cp $installdir/bin/columnstore.def /etc/default/columnstore + + sed -i -e s@prefix=/usr/local@prefix=$HOME@g $installdir/bin/* + fi #check if MariaDB Columnstore system logging was setup diff --git a/oam/install_scripts/post-mysqld-install b/oam/install_scripts/post-mysqld-install index 833f5f26c..546945b22 100755 --- a/oam/install_scripts/post-mysqld-install +++ b/oam/install_scripts/post-mysqld-install @@ -82,9 +82,17 @@ chown -R $user.$user $installdir/mysql if [ -f $installdir/lib/libcalmysql.so.1.0.0 ]; then libcalmysql=$installdir/lib/libcalmysql.so.1.0.0 libudfsdk=$installdir/lib/libudf_mysql.so.1.0.0 + is_columnstore_tables=$installdir/lib/is_columnstore_tables.so.1.0.0 + is_columnstore_columns=$installdir/lib/is_columnstore_columns.so.1.0.0 + is_columnstore_extents=$installdir/lib/is_columnstore_extents.so.1.0.0 + is_columnstore_files=$installdir/lib/is_columnstore_files.so.1.0.0 elif [ -f $installdir/lib/libcalmysql.so.1 ]; then libcalmysql=$installdir/lib/libcalmysql.so.1 libudfsdk=$installdir/lib/libudf_mysql.so.1 + is_columnstore_tables=$installdir/lib/is_columnstore_tables.so.1 + is_columnstore_columns=$installdir/lib/is_columnstore_columns.so.1 + is_columnstore_extents=$installdir/lib/is_columnstore_extents.so.1 + is_columnstore_files=$installdir/lib/is_columnstore_files.so.1 else libcalmysql= fi @@ -93,8 +101,12 @@ if [ -n "$libcalmysql" ]; then cd $installdir/mysql/lib/plugin ln -sf $libcalmysql libcalmysql.so - ln -sf $libcalmysql libcalmysqlent.so + ln -sf $libcalmysql libcalmysqlent.so ln -sf $libudfsdk libudf_mysql.so + ln -sf $is_columnstore_tables is_columnstore_tables.so + ln -sf $is_columnstore_columns is_columnstore_columns.so + ln -sf $is_columnstore_extents is_columnstore_extents.so + ln -sf $is_columnstore_files is_columnstore_files.so fi # cleanup previous install mysql replication files diff --git a/oam/install_scripts/pre-uninstall b/oam/install_scripts/pre-uninstall index 464071188..43db84d41 100755 --- a/oam/install_scripts/pre-uninstall +++ b/oam/install_scripts/pre-uninstall @@ -33,22 +33,27 @@ for arg in "$@"; do fi done +if [ $user = "root" ]; then + . /root/.bashrc +else + . $HOME/.bashrc +fi + #stop services $installdir/bin/columnstore stop > /dev/null 2>&1 $installdir/myql/columnstore-Mysql stop > /dev/null 2>&1 # uninstall OS scripts updated by postConfigure -if test -f /root/.bashrc.columnstoreSave ; then - $sudo mv -f /root/.bashrc.columnstoreSave /root/.bashrc > /dev/null 2>&1 -fi if test -f /etc/exports.columnstoreSave ; then $sudo mv -f /etc/exports.columnstoreSave /etc/exports > /dev/null 2>&1 else $sudo rm -f /etc/exports > /dev/null 2>&1 fi -if test -f /etc/rc.d/rc.local.columnstoreSave ; then + +if test -f /etc/rc.d/rc.local.columnstoreSave ; then $sudo mv -f /etc/rc.d/rc.local.columnstoreSave /etc/rc.d/rc.local > /dev/null 2>&1 fi + if test -f /etc/rc.local.columnstoreSave ; then $sudo mv -f /etc/rc.local.columnstoreSave /etc/rc.local > /dev/null 2>&1 fi @@ -65,23 +70,23 @@ fi #remove log file directories #rm -rf /var/log/mariadb/columnstore > /dev/null 2>&1 #rm -f $installdir/mysql/db/*.err > /dev/null 2>&1 -rm -f /var/log/mariadb/columnstore/activeAlarms > /dev/null 2>&1 -rm -f /var/log/mariadb/columnstore/*.log1 > /dev/null 2>&1 +sudo rm -f /var/log/mariadb/columnstore/activeAlarms > /dev/null 2>&1 +sudo rm -f /var/log/mariadb/columnstore/*.log1 > /dev/null 2>&1 rm -rf $installdir/mysql/db/columnstore_log_archive > /dev/null 2>&1 sudo rm -f /etc/default/columnstore # remove Mariab Columnstore Log Rotate File abd Transaction Log File -rm -f /etc/logrotate.d/columnstore > /dev/null 2>&1 -rm -f /etc/cron.d/transactionLog > /dev/null 2>&1 +sudo rm -f /etc/logrotate.d/columnstore > /dev/null 2>&1 +sudo rm -f /etc/cron.d/transactionLog > /dev/null 2>&1 # delete Mariab Columnstore shared memory segments $installdir/bin/clearShm > /dev/null 2>&1 # delete prat files -rm -f /etc/cron.d/ps > /dev/null 2>&1 -rm -f /etc/pscollect > /dev/null 2>&1 -/etc/init.d/crond reload > /dev/null 2>&1 -sudo systemctl reload crond.service > /dev/null 2>&1 +sudo rm -f /etc/cron.d/ps > /dev/null 2>&1 +sudo rm -f /etc/pscollect > /dev/null 2>&1 +sudo /etc/init.d/crond reload > /dev/null 2>&1 +sudo sudo systemctl reload crond.service > /dev/null 2>&1 # delete tmp files rm -f $installdir/local/*.columnstore @@ -90,7 +95,7 @@ rm -rf /tmp/bucketreuse rm -f /tmp/columnstore.txt rm -f /tmp/dbbuilder.* rm -f /tmp/dbrmfiles -rm -f /var/lock/subsys/columnstore +sudo rm -f /var/lock/subsys/columnstore rm -f /tmp/pkgcheck rm -f /tmp/upgrade-status.log.* rm -f /tmp/mount.log @@ -122,8 +127,8 @@ systemctl=`which systemctl 2>/dev/null` if [ -n "$systemctl" ]; then sudo systemctl disable columnstore >/dev/null 2>&1 - rm -f /usr/lib/systemd/system/columnstore.service - rm -f /lib/systemd/system/columnstore.service + sudo rm -f /usr/lib/systemd/system/columnstore.service + sudo rm -f /lib/systemd/system/columnstore.service else chkconfig=`which chkconfig 2>/dev/null` if [ -n "$chkconfig" ]; then @@ -158,9 +163,20 @@ fi #remove library paths if [ $installdir == "/usr/local/mariadb/columnstore" ]; then # remove library config file - rm -f /etc/ld.so.conf.d/columnstore.conf - rm -f /etc/ld.so.cache - ldconfig + sudo rm -f /etc/ld.so.conf.d/columnstore.conf + sudo rm -f /etc/ld.so.cache + sudo ldconfig +fi + +#cleanup .bashrc +if [ $user = "root" ]; then + if test -f /root/.bashrc.columnstoreSave ; then + mv -f /root/.bashrc.columnstoreSave /root/.bashrc > /dev/null 2>&1 + fi +else + if test -f $HOME/.bashrc.columnstoreSave ; then + mv -f /home/$user/.bashrc.columnstoreSave /home/$user/.bashrc > /dev/null 2>&1 + fi fi #tell user to run post configure script diff --git a/oam/install_scripts/rsync.sh b/oam/install_scripts/rsync.sh index 2fa53ee6b..d7d73d2d0 100755 --- a/oam/install_scripts/rsync.sh +++ b/oam/install_scripts/rsync.sh @@ -6,7 +6,6 @@ # Argument 1 - Remote Server Host Name or IP address # Argument 2 - Root Password of remote server # Argument 3 - Debug flag 1 for on, 0 for off -set USERNAME "root@" set SERVER [lindex $argv 0] set PASSWORD [lindex $argv 1] set INSTALLDIR [lindex $argv 2] @@ -15,13 +14,19 @@ set DEBUG [lindex $argv 3] log_user $DEBUG spawn -noecho /bin/bash +if {[info exists env(USER)]} { + set USERNAME $env(USER) +} else { + set USERNAME "root" +} + if { $PASSWORD == "ssh" } { set USERNAME "" set PASSWORD "" } -set COMMAND "rsync -vopgr -e ssh --exclude=mysql/ --exclude=test/ --exclude=infinidb_vtable/ --exclude=infinidb_querystats/ --exclude=calpontsys/ --include=*/ --include=*/* --exclude=* $INSTALLDIR/mysql/db/ $USERNAME$SERVER:$INSTALLDIR/mysql/db/" +set COMMAND "rsync -vopgr -e ssh --exclude=mysql/ --exclude=test/ --exclude=infinidb_vtable/ --exclude=infinidb_querystats/ --exclude=calpontsys/ --include=*/ --include=*/* --exclude=* $INSTALLDIR/mysql/db/ $USERNAME@$SERVER:$INSTALLDIR/mysql/db/" # # run command diff --git a/oam/install_scripts/slave-rep-columnstore.sh b/oam/install_scripts/slave-rep-columnstore.sh index 000dc532a..efd6609f2 100644 --- a/oam/install_scripts/slave-rep-columnstore.sh +++ b/oam/install_scripts/slave-rep-columnstore.sh @@ -74,7 +74,6 @@ CHANGE MASTER TO MASTER_PORT=$port, MASTER_LOG_FILE='$masterlogfile', MASTER_LOG_POS=$masterlogpos; - MASTER_USE_GTID=$masterlogpos; EOD diff --git a/oam/install_scripts/syslogSetup.sh b/oam/install_scripts/syslogSetup.sh index 73315d1b6..ae606f3d4 100755 --- a/oam/install_scripts/syslogSetup.sh +++ b/oam/install_scripts/syslogSetup.sh @@ -5,13 +5,21 @@ # syslogSetup.sh - install / uninstall MariaDB Columnstore system logging configuration # no point in going any further if not root... (only works in bash) -test $EUID -eq 0 || exit 0 +#test $EUID -eq 0 || exit 0 prefix=/usr/local installdir=$prefix/mariadb/columnstore syslog_conf=nofile rsyslog7=0 +user=$USER +sudo=sudo +if [ -z "$user" ]; then + user=root + sudo=" " +fi + + for arg in "$@"; do if [ `expr -- "$arg" : '--prefix='` -eq 9 ]; then prefix="`echo $arg | awk -F= '{print $2}'`" @@ -62,13 +70,13 @@ if [ "$daemon" = "nodaemon" ]; then if [ -f /etc/syslog.conf ]; then daemon="syslog" - /etc/init.d/syslog start > /dev/null 2>&1 + sudo /etc/init.d/syslog start > /dev/null 2>&1 elif [ -f /etc/rsyslog.conf ]; then daemon="rsyslog" - /etc/init.d/rsyslog start > /dev/null 2>&1 + sudo /etc/init.d/rsyslog start > /dev/null 2>&1 elif [ -f /etc/init.d/syslog-ng ]; then daemon="syslog-ng" - /etc/init.d/syslog-ng start > /dev/null 2>&1 + sudo /etc/init.d/syslog-ng start > /dev/null 2>&1 fi fi @@ -92,7 +100,7 @@ if [ "$daemon" = "syslog-ng" ]; then fi elif [ "$daemon" = "rsyslog" ]; then #check if rsyslog version 7 or greater - rsyslogd -v > /tmp/rsyslog.ver + sudo rsyslogd -v > /tmp/rsyslog.ver cnt=`grep "rsyslogd 7" /tmp/rsyslog.ver | wc -l` if [ $cnt -gt 0 ]; then rsyslog7=1 @@ -149,30 +157,33 @@ checkSyslog if [ ! -z "$syslog_conf" ] ; then $installdir/bin/setConfig -d Installation SystemLogConfigFile ${syslog_conf} >/dev/null 2>&1 if [ "$syslog_conf" != /etc/rsyslog.d/columnstore.conf ]; then - rm -f ${syslog_conf}.columnstoreSave - cp ${syslog_conf} ${syslog_conf}.columnstoreSave >/dev/null 2>&1 - sed -i '/# MariaDB/,$d' ${syslog_conf}.columnstoreSave > /dev/null 2>&1 + sudo rm -f ${syslog_conf}.columnstoreSave + sudo cp ${syslog_conf} ${syslog_conf}.columnstoreSave >/dev/null 2>&1 + sudo sed -i '/# MariaDB/,$d' ${syslog_conf}.columnstoreSave > /dev/null 2>&1 fi egrep -qs 'MariaDB Columnstore Database Platform Logging' ${syslog_conf} if [ $? -ne 0 ]; then - #set the syslog for calpont logging + #set the syslog for ColumnStore logging # remove older version incase it was installed by previous build - rm -rf /etc/rsyslog.d/columnstore.conf + sudo rm -rf /etc/rsyslog.d/columnstore.conf if [ $rsyslog7 == 1 ]; then - rm -f /etc/rsyslog.d/49-columnstore.conf - cat ${columnstoreSyslogFile7} >> ${syslog_conf} - chown syslog:adm /var/log/mariadb/columnstore + sudo rm -f /etc/rsyslog.d/49-columnstore.conf + sudo cp ${columnstoreSyslogFile7} ${syslog_conf} + sudo chown syslog:adm /var/log/mariadb/columnstore >/dev/null 2>&1 else - cat ${columnstoreSyslogFile} >> ${syslog_conf} + sudo cp ${columnstoreSyslogFile} ${syslog_conf} fi fi - pkill -hup syslogd > /dev/null 2>&1 - pkill -hup syslog-ng > /dev/null 2>&1 - pkill -hup rsyslogd > /dev/null 2>&1 - /etc/init.d/rsyslog restart > /dev/null 2>&1 + sudo etc/init.d/rsyslog restart > /dev/null 2>&1 + sudo /etc/init.d/syslog restart > /dev/null 2>&1 + sudo /etc/init.d/syslog-ng restart > /dev/null 2>&1 + systemctl restart rsyslog.service > /dev/null 2>&1 + systemctl restart syslog.service > /dev/null 2>&1 + systemctl restart syslog-ng.service > /dev/null 2>&1 + fi } @@ -185,26 +196,30 @@ if [ ! -z "$syslog_conf" ] ; then egrep -qs 'MariaDB Columnstore Database Platform Logging' ${syslog_conf} if [ $? -eq 0 ]; then if [ -f ${syslog_conf}.columnstoreSave ] ; then - #uninstall the syslog for calpont logging - mv -f ${syslog_conf} ${syslog_conf}.calpontBackup - mv -f ${syslog_conf}.columnstoreSave ${syslog_conf} >/dev/null 2>&1 + #uninstall the syslog for ColumnStore logging + sudo v -f ${syslog_conf} ${syslog_conf}.ColumnStoreBackup + sudo mv -f ${syslog_conf}.columnstoreSave ${syslog_conf} >/dev/null 2>&1 if [ ! -f ${syslog_conf} ] ; then - cp ${syslog_conf}.calpontBackup ${syslog_conf} + sudo cp ${syslog_conf}.ColumnStoreBackup ${syslog_conf} fi fi fi - sed -i '/# MariaDB/,$d' ${syslog_conf} > /dev/null 2>&1 + sudo sed -i '/# MariaDB/,$d' ${syslog_conf} > /dev/null 2>&1 else - rm -f "$syslog_conf" + sudo rm -f "$syslog_conf" fi else - rm -f "$syslog_conf" + sudo rm -f "$syslog_conf" fi - pkill -hup syslogd > /dev/null 2>&1 - pkill -hup syslog-ng > /dev/null 2>&1 - /etc/init.d/rsyslog restart > /dev/null 2>&1 - systemctl restart rsyslog.service > /dev/null 2>&1 + sudo etc/init.d/rsyslog restart > /dev/null 2>&1 + sudo /etc/init.d/syslog restart > /dev/null 2>&1 + sudo /etc/init.d/syslog-ng restart > /dev/null 2>&1 + + systemctl restart rsyslog.service > /dev/null 2>&1 + systemctl restart syslog.service > /dev/null 2>&1 + systemctl restart syslog-ng.service > /dev/null 2>&1 + $installdir/bin/setConfig -d Installation SystemLogConfigFile "unassigned" @@ -227,7 +242,7 @@ fi check() { test -f $installdir/post/functions && . $installdir/post/functions number=$RANDOM -cplogger -i 104 "MariaDB Columnstore Log Test: $number" +$installdir/bin/cplogger -i 104 "MariaDB Columnstore Log Test: $number" sleep 3 egrep -qs "MariaDB Columnstore Log Test: $number" /var/log/mariadb/columnstore/info.log if [ $? -eq 0 ]; then diff --git a/oam/oamcpp/liboamcpp.cpp b/oam/oamcpp/liboamcpp.cpp index efd2cc977..19ed72529 100644 --- a/oam/oamcpp/liboamcpp.cpp +++ b/oam/oamcpp/liboamcpp.cpp @@ -1367,13 +1367,13 @@ namespace oam { processor.shutdown(); string error = e.what(); - writeLog("getSystemStatus: write exception: " + error, LOG_TYPE_ERROR); + //writeLog("getSystemStatus: write exception: " + error, LOG_TYPE_ERROR); exceptionControl("getSystemStatus write", API_FAILURE); } catch(...) { processor.shutdown(); - writeLog("getSystemStatus: write exception: unknown", LOG_TYPE_ERROR); + //writeLog("getSystemStatus: write exception: unknown", LOG_TYPE_ERROR); exceptionControl("getSystemStatus write", API_FAILURE); } @@ -1386,13 +1386,13 @@ namespace oam { processor.shutdown(); string error = e.what(); - writeLog("getSystemStatus: read exception: " + error, LOG_TYPE_ERROR); + //writeLog("getSystemStatus: read exception: " + error, LOG_TYPE_ERROR); exceptionControl("getSystemStatus read", API_FAILURE); } catch(...) { processor.shutdown(); - writeLog("getSystemStatus: read exception: unknown", LOG_TYPE_ERROR); + //writeLog("getSystemStatus: read exception: unknown", LOG_TYPE_ERROR); exceptionControl("getSystemStatus read", API_FAILURE); } @@ -1475,21 +1475,21 @@ namespace oam } else { - writeLog("getSystemStatus: ProcStatusControl returns 0 length", LOG_TYPE_ERROR); + //writeLog("getSystemStatus: ProcStatusControl returns 0 length", LOG_TYPE_ERROR); } // timeout ocurred, shutdown connection processor.shutdown(); - writeLog("getSystemStatus: read 0 length", LOG_TYPE_ERROR); + //writeLog("getSystemStatus: read 0 length", LOG_TYPE_ERROR); exceptionControl("getSystemStatus read 0", API_FAILURE); } catch (exception& e) { string error = e.what(); - writeLog("getSystemStatus: final exception: " + error, LOG_TYPE_ERROR); + //writeLog("getSystemStatus: final exception: " + error, LOG_TYPE_ERROR); } catch(...) { - writeLog("getSystemStatus: final exception: unknown", LOG_TYPE_ERROR); + //writeLog("getSystemStatus: final exception: unknown", LOG_TYPE_ERROR); } } @@ -1564,13 +1564,13 @@ namespace oam Oam oam; ostringstream os; os << "Oam::getModuleStatus exception while getNICStatus " << (*pt1).HostName << " " << e.what(); - oam.writeLog(os.str(), logging::LOG_TYPE_ERROR); + //oam.writeLog(os.str(), logging::LOG_TYPE_ERROR); } catch (...) { Oam oam; ostringstream os; os << "Oam::getModuleStatus exception while getNICStatus " << (*pt1).HostName; - oam.writeLog(os.str(), logging::LOG_TYPE_ERROR); + //oam.writeLog(os.str(), logging::LOG_TYPE_ERROR); } } @@ -1589,13 +1589,13 @@ namespace oam Oam oam; ostringstream os; os << "Oam::getModuleStatus exception while getSystemConfig " << name << " " << e.what(); - oam.writeLog(os.str(), logging::LOG_TYPE_ERROR); + //oam.writeLog(os.str(), logging::LOG_TYPE_ERROR); } catch (...) { Oam oam; ostringstream os; os << "Oam::getModuleStatus exception while getSystemConfig " << name; - oam.writeLog(os.str(), logging::LOG_TYPE_ERROR); + //oam.writeLog(os.str(), logging::LOG_TYPE_ERROR); } } } @@ -1605,13 +1605,13 @@ namespace oam Oam oam; ostringstream os; os << "Oam::getModuleStatus exception while getSystemStatus " << e.what(); - oam.writeLog(os.str(), logging::LOG_TYPE_ERROR); + //oam.writeLog(os.str(), logging::LOG_TYPE_ERROR); } catch (...) { Oam oam; ostringstream os; os << "Oam::getModuleStatus exception while getSystemStatus"; - oam.writeLog(os.str(), logging::LOG_TYPE_ERROR); + //oam.writeLog(os.str(), logging::LOG_TYPE_ERROR); } // no match found @@ -1795,7 +1795,7 @@ namespace oam Oam oam; ostringstream os; os << "Oam::getNICStatus exception while getSystemStatus for " << name << " " << e.what(); - oam.writeLog(os.str(), logging::LOG_TYPE_ERROR); + //oam.writeLog(os.str(), logging::LOG_TYPE_ERROR); exceptionControl("getNICStatus", API_FAILURE); } @@ -5984,7 +5984,15 @@ namespace oam //format attached volume writeLog("addUMdisk - Format new Volume for: " + volumeName, LOG_TYPE_DEBUG); - string cmd = "mkfs.ext2 -F " + device + " > /dev/null 2>&1"; + + string cmd; + int user; + user = getuid(); + if (user == 0) + cmd = "mkfs.ext2 -F " + device + " > /dev/null 2>&1"; + else + cmd = "sudo mkfs.ext2 -F " + device + " > /dev/null 2>&1"; + system(cmd.c_str()); //detach volume @@ -6202,7 +6210,13 @@ namespace oam //format attached volume cout << " Formatting DBRoot #" << itoa(*pt1) << ", please wait..." << endl; - string cmd = "mkfs.ext2 -F " + amazonDeviceName + " > /tmp/format.log 2>&1"; + string cmd; + int user; + user = getuid(); + if (user == 0) + cmd = "mkfs.ext2 -F " + amazonDeviceName + " > /tmp/format.log 2>&1"; + else + cmd = "sudo mkfs.ext2 -F " + amazonDeviceName + " > /tmp/format.log 2>&1"; writeLog("addDbroot format cmd: " + cmd, LOG_TYPE_DEBUG ); @@ -7567,7 +7581,7 @@ namespace oam std::string Oam::getEC2InstanceIpAddress(std::string instanceName) { // run script to get Instance status and IP Address - string cmd = InstallDir + "/bin/IDBInstanceCmds.sh getPrivateIP " + instanceName + " > /tmp/getCloudIP_" + instanceName; + string cmd = InstallDir + "/bin/MCSInstanceCmds.sh getPrivateIP " + instanceName + " > /tmp/getCloudIP_" + instanceName; system(cmd.c_str()); if (checkLogStatus("/tmp/getCloudIP_" + instanceName, "stopped") ) @@ -7604,7 +7618,7 @@ namespace oam std::string Oam::getEC2LocalInstance(std::string name) { // run script to get Instance status and IP Address - string cmd = InstallDir + "/bin/IDBInstanceCmds.sh getInstance > /tmp/getInstanceInfo_" + name; + string cmd = InstallDir + "/bin/MCSInstanceCmds.sh getInstance > /tmp/getInstanceInfo_" + name; int status = system(cmd.c_str()); if (WEXITSTATUS(status) != 0 ) return "failed"; @@ -7635,7 +7649,7 @@ namespace oam std::string Oam::getEC2LocalInstanceType(std::string name) { // run script to get Instance status and IP Address - string cmd = InstallDir + "/bin/IDBInstanceCmds.sh getType > /tmp/getInstanceType_" + name; + string cmd = InstallDir + "/bin/MCSInstanceCmds.sh getType > /tmp/getInstanceType_" + name; int status = system(cmd.c_str()); if (WEXITSTATUS(status) != 0 ) return "failed"; @@ -7666,7 +7680,7 @@ namespace oam std::string Oam::getEC2LocalInstanceSubnet(std::string name) { // run script to get Instance Subnet - string cmd = InstallDir + "/bin/IDBInstanceCmds.sh getSubnet > /tmp/getInstanceSubnet_" + name; + string cmd = InstallDir + "/bin/MCSInstanceCmds.sh getSubnet > /tmp/getInstanceSubnet_" + name; int status = system(cmd.c_str()); if (WEXITSTATUS(status) != 0 ) return "failed"; @@ -7698,7 +7712,7 @@ namespace oam std::string Oam::launchEC2Instance( const std::string name, const std::string IPAddress, const std::string type, const std::string group) { // run script to get Instance status and IP Address - string cmd = InstallDir + "/bin/IDBInstanceCmds.sh launchInstance " + IPAddress + " " + type + " " + group + " > /tmp/getInstance_" + name; + string cmd = InstallDir + "/bin/MCSInstanceCmds.sh launchInstance " + IPAddress + " " + type + " " + group + " > /tmp/getInstance_" + name; int status = system(cmd.c_str()); if (WEXITSTATUS(status) != 0 ) return "failed"; @@ -7740,7 +7754,7 @@ namespace oam void Oam::terminateEC2Instance(std::string instanceName) { // run script to get Instance status and IP Address - string cmd = InstallDir + "/bin/IDBInstanceCmds.sh terminateInstance " + instanceName + " > /tmp/terminateEC2Instance_" + instanceName; + string cmd = InstallDir + "/bin/MCSInstanceCmds.sh terminateInstance " + instanceName + " > /tmp/terminateEC2Instance_" + instanceName; system(cmd.c_str()); return; @@ -7757,7 +7771,7 @@ namespace oam void Oam::stopEC2Instance(std::string instanceName) { // run script to get Instance status and IP Address - string cmd = InstallDir + "/bin/IDBInstanceCmds.sh stopInstance " + instanceName + " > /tmp/stopEC2Instance_" + instanceName; + string cmd = InstallDir + "/bin/MCSInstanceCmds.sh stopInstance " + instanceName + " > /tmp/stopEC2Instance_" + instanceName; system(cmd.c_str()); return; @@ -7774,7 +7788,7 @@ namespace oam bool Oam::startEC2Instance(std::string instanceName) { // run script to get Instance status and IP Address - string cmd = InstallDir + "/bin/IDBInstanceCmds.sh startInstance " + instanceName + " > /tmp/startEC2Instance_" + instanceName; + string cmd = InstallDir + "/bin/MCSInstanceCmds.sh startInstance " + instanceName + " > /tmp/startEC2Instance_" + instanceName; int ret = system(cmd.c_str()); if (WEXITSTATUS(ret) != 0 ) return false; @@ -7793,7 +7807,7 @@ namespace oam bool Oam::assignElasticIP(std::string instanceName, std::string IpAddress) { // run script to get Instance status and IP Address - string cmd = InstallDir + "/bin/IDBInstanceCmds.sh assignElasticIP " + instanceName + " " + IpAddress + " > /tmp/assignElasticIP_" + instanceName; + string cmd = InstallDir + "/bin/MCSInstanceCmds.sh assignElasticIP " + instanceName + " " + IpAddress + " > /tmp/assignElasticIP_" + instanceName; int ret = system(cmd.c_str()); if (WEXITSTATUS(ret) != 0 ) exceptionControl("assignElasticIP", oam::API_FAILURE); @@ -7812,7 +7826,7 @@ namespace oam bool Oam::deassignElasticIP(std::string IpAddress) { // run script to get Instance status and IP Address - string cmd = InstallDir + "/bin/IDBInstanceCmds.sh deassignElasticIP " + IpAddress + " > /tmp/deassignElasticIP_" + IpAddress; + string cmd = InstallDir + "/bin/MCSInstanceCmds.sh deassignElasticIP " + IpAddress + " > /tmp/deassignElasticIP_" + IpAddress; int ret = system(cmd.c_str()); if (WEXITSTATUS(ret) != 0 ) exceptionControl("deassignElasticIP", oam::API_FAILURE); @@ -7831,7 +7845,7 @@ namespace oam std::string Oam::getEC2VolumeStatus(std::string volumeName) { // run script to get Volume Status - string cmd = InstallDir + "/bin/IDBVolumeCmds.sh describe " + volumeName + " > /tmp/getVolumeStatus_" + volumeName; + string cmd = InstallDir + "/bin/MCSVolumeCmds.sh describe " + volumeName + " > /tmp/getVolumeStatus_" + volumeName; int ret = system(cmd.c_str()); if (WEXITSTATUS(ret) != 0 ) return "failed"; @@ -7862,7 +7876,7 @@ namespace oam std::string Oam::createEC2Volume(std::string size, std::string name) { // run script to get Volume Status - string cmd = InstallDir + "/bin/IDBVolumeCmds.sh create " + size + " " + name + " > /tmp/createVolumeStatus_" + name; + string cmd = InstallDir + "/bin/MCSVolumeCmds.sh create " + size + " " + name + " > /tmp/createVolumeStatus_" + name; int ret = system(cmd.c_str()); if (WEXITSTATUS(ret) != 0 ) return "failed"; @@ -7906,7 +7920,7 @@ namespace oam for ( int retry = 0 ; retry < 2 ; retry++ ) { // run script to attach Volume - string cmd = InstallDir + "/bin/IDBVolumeCmds.sh attach " + volumeName + " " + instanceName + " " + deviceName + " > /tmp/attachVolumeStatus_" + volumeName; + string cmd = InstallDir + "/bin/MCSVolumeCmds.sh attach " + volumeName + " " + instanceName + " " + deviceName + " > /tmp/attachVolumeStatus_" + volumeName; ret = system(cmd.c_str()); if (WEXITSTATUS(ret) == 0 ) @@ -7933,7 +7947,7 @@ namespace oam bool Oam::detachEC2Volume(std::string volumeName) { // run script to attach Volume - string cmd = InstallDir + "/bin/IDBVolumeCmds.sh detach " + volumeName + " > /tmp/detachVolumeStatus_" + volumeName; + string cmd = InstallDir + "/bin/MCSVolumeCmds.sh detach " + volumeName + " > /tmp/detachVolumeStatus_" + volumeName; int ret = system(cmd.c_str()); if (WEXITSTATUS(ret) != 0 ) return false; @@ -7952,7 +7966,7 @@ namespace oam bool Oam::deleteEC2Volume(std::string volumeName) { // run script to delete Volume - string cmd = InstallDir + "/bin/IDBVolumeCmds.sh delete " + volumeName + " > /tmp/deleteVolumeStatus_" + volumeName; + string cmd = InstallDir + "/bin/MCSVolumeCmds.sh delete " + volumeName + " > /tmp/deleteVolumeStatus_" + volumeName; int ret = system(cmd.c_str()); if (WEXITSTATUS(ret) != 0 ) return false; @@ -7971,7 +7985,7 @@ namespace oam bool Oam::createEC2tag(std::string resourceName, std::string tagName, std::string tagValue) { // run script to create a tag - string cmd = InstallDir + "/bin/IDBVolumeCmds.sh createTag " + resourceName + " " + tagName + " " + tagValue + " > /tmp/createTagStatus_" + resourceName; + string cmd = InstallDir + "/bin/MCSVolumeCmds.sh createTag " + resourceName + " " + tagName + " " + tagValue + " > /tmp/createTagStatus_" + resourceName; int ret = system(cmd.c_str()); if (WEXITSTATUS(ret) != 0 ) return false; @@ -8540,17 +8554,32 @@ namespace oam { writeLog("updateFstab called: " + device + ":" + dbrootID, LOG_TYPE_DEBUG ); - //check if entry already exist - string cmd = "grep /data" + dbrootID + " /etc/fstab > /dev/null 2>&1"; + //check if entry already exist + int user; + user = getuid(); + + string cmd; + if (user == 0) + cmd = "grep /data" + dbrootID + " /etc/fstab > /dev/null 2>&1"; + else + cmd = "sudo grep /data" + dbrootID + " /etc/fstab > /dev/null 2>&1"; + int status = system(cmd.c_str()); if (WEXITSTATUS(status) == 0 ) return ""; //update /etc/fstab with mount - string entry = device + " " + InstallDir + "/data" + dbrootID + " ext2 noatime,nodiratime,noauto 0 0"; + string entry; + if (user == 0) + entry = device + " " + InstallDir + "/data" + dbrootID + " ext2 noatime,nodiratime,noauto 0 0"; + else + entry = device + " " + InstallDir + "/data" + dbrootID + " ext2 noatime,nodiratime,noauto,user 0 0"; - //update local fstab - cmd = "echo " + entry + " >> /etc/fstab"; + //update local fstab + if (user == 0) + cmd = "echo " + entry + " >> /etc/fstab"; + else + cmd = "sudo echo " + entry + " >> /etc/fstab"; system(cmd.c_str()); //use from addmodule later @@ -9614,7 +9643,7 @@ namespace oam // not root user // The stat above may fail for non-root because of permissions // This is a non-optimal solution - string cmd = "pgrep ProcMon"; + string cmd = "pgrep ProcMon > /dev/null 2>&1"; if (system(cmd.c_str()) == 0) { return true; diff --git a/oamapps/alarmmanager/alarmmanager.cpp b/oamapps/alarmmanager/alarmmanager.cpp index 499cc7755..7d33b0302 100644 --- a/oamapps/alarmmanager/alarmmanager.cpp +++ b/oamapps/alarmmanager/alarmmanager.cpp @@ -234,7 +234,6 @@ void processAlarm(const Alarm& calAlarm) rewriteActiveLog (alarmList); } catch (runtime_error& e) { - if (ALARM_DEBUG) { LoggingID lid(11); MessageLog ml(lid); Message msg; @@ -242,9 +241,7 @@ void processAlarm(const Alarm& calAlarm) args.add("rewriteActiveLog error:"); args.add(e.what()); msg.format(args); - ml.logDebugMessage(msg); - } - exit(1); + ml.logErrorMessage(msg); } break; } @@ -256,7 +253,6 @@ void processAlarm(const Alarm& calAlarm) logAlarm (calAlarm, ACTIVE_ALARM_FILE); } catch (runtime_error& e) { - if (ALARM_DEBUG) { LoggingID lid(11); MessageLog ml(lid); Message msg; @@ -264,9 +260,7 @@ void processAlarm(const Alarm& calAlarm) args.add("logAlarm error:"); args.add(e.what()); msg.format(args); - ml.logDebugMessage(msg); - } - exit(1); + ml.logErrorMessage(msg); } } @@ -276,7 +270,6 @@ void processAlarm(const Alarm& calAlarm) logAlarm (calAlarm, ALARM_FILE); } catch (runtime_error& e) { - if (ALARM_DEBUG) { LoggingID lid(11); MessageLog ml(lid); Message msg; @@ -284,9 +277,7 @@ void processAlarm(const Alarm& calAlarm) args.add("logAlarm error:"); args.add(e.what()); msg.format(args); - ml.logDebugMessage(msg); - } - exit(1); + ml.logErrorMessage(msg); } } } @@ -462,7 +453,6 @@ void ALARMManager::sendAlarmReport (const char* componentID, int alarmID, int st configAlarm (calAlarm); } catch (runtime_error& e) { - if (ALARM_DEBUG) { LoggingID lid(11); MessageLog ml(lid); Message msg; @@ -470,9 +460,7 @@ void ALARMManager::sendAlarmReport (const char* componentID, int alarmID, int st args.add("configAlarm error:"); args.add(e.what()); msg.format(args); - ml.logDebugMessage(msg); - } - exit(1); + ml.logErrorMessage(msg); } return; diff --git a/oamapps/mcsadmin/CMakeLists.txt b/oamapps/mcsadmin/CMakeLists.txt index 5d06594df..c8d4af0e2 100644 --- a/oamapps/mcsadmin/CMakeLists.txt +++ b/oamapps/mcsadmin/CMakeLists.txt @@ -8,7 +8,7 @@ set(mcsadmin_SRCS mcsadmin.cpp) add_executable(mcsadmin ${mcsadmin_SRCS}) -target_link_libraries(mcsadmin ${ENGINE_LDFLAGS} readline ncurses ${ENGINE_EXEC_LIBS}) +target_link_libraries(mcsadmin ${ENGINE_LDFLAGS} readline ncurses ${ENGINE_EXEC_LIBS} ${ENGINE_WRITE_LIBS}) install(TARGETS mcsadmin DESTINATION ${ENGINE_BINDIR} COMPONENT platform) diff --git a/oamapps/mcsadmin/mcsadmin.cpp b/oamapps/mcsadmin/mcsadmin.cpp index b2c3e27b4..100be2af8 100644 --- a/oamapps/mcsadmin/mcsadmin.cpp +++ b/oamapps/mcsadmin/mcsadmin.cpp @@ -36,8 +36,8 @@ extern int h_errno; #include "messagequeue.h" #include "we_messages.h" #include "we_redistributedef.h" - - +#include "we_config.h" // for findObjectFile +#include "we_fileop.h" // for findObjectFile namespace fs = boost::filesystem; using namespace alarmmanager; @@ -46,6 +46,7 @@ using namespace oam; using namespace config; using namespace messageqcpp; using namespace redistribute; +using namespace execplan; #include "installdir.h" @@ -860,8 +861,95 @@ int processCommand(string* arguments) } break; - case 5: // Available + case 5: // findObjectFile { + unsigned maxDBRoot = WriteEngine::Config::DBRootCount(); + if (maxDBRoot < 1) + { + cout << endl << "getDatafileName fails because there are no dbroots defined for this server" << endl; + break;; + } + if (arguments[1] == "") + { + cout << endl << "getDatafileName requires one of" << endl; + cout << "a) oid of column for which file name is to be retrieved" << endl; + cout << "b) schema, table and column for which file name is to be retrieved" << endl; + break; + } + char* endchar; + int oid = strtol(arguments[1].c_str(), &endchar, 0); + // test to see if not all numeric + if (endchar < &(*arguments[1].end())) + { + oid = 0; + } + if (oid == 0) + { + // Need to convert the arguments to oid + boost::shared_ptr systemCatalogPtr = + execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(0); + CalpontSystemCatalog::TableColName columnName; + columnName.schema = arguments[1]; + if (arguments[2] == "") + { + cout << endl << "getDatafileName requires a table and column for schema " << arguments[1] << endl; + break; + } + columnName.table = arguments[2]; + if (arguments[3] == "") + { + // No column was given. Use the first column in the table. + CalpontSystemCatalog::TableName tableName; + tableName.schema = arguments[1]; + tableName.table = arguments[2]; + CalpontSystemCatalog::RIDList rdlist = systemCatalogPtr->columnRIDs(tableName); + oid = rdlist.front().objnum; + } + else + { + columnName.column = arguments[3]; + oid = systemCatalogPtr->lookupOID(columnName); + } + } + + // Use writeengine code to get the filename + WriteEngine::FileOp fileOp; + char fileName[WriteEngine::FILE_NAME_SIZE]; + memset(fileName, 0, WriteEngine::FILE_NAME_SIZE); + int rc; + + if (oid < 1000) + rc = fileOp.getVBFileName(oid, fileName); + else + rc = fileOp.oid2DirName(oid, fileName); + cout << "file name for oid " << oid << ":" << endl; + if (strlen(fileName) > 0) + { + cout << fileName; + } + if (rc == WriteEngine::NO_ERROR) + { + // Success. No more output. + cout << endl; + } + else if (rc == WriteEngine::ERR_FILE_NOT_EXIST) + { + if (strlen(fileName) == 0) + { + // We couldn't get a name + cout << "Error: Filename could not be determined" << endl; + } + else + { + // We got a name, but the file doesn't exist + cout << " (OID directory not found)" << endl; + } + } + else + { + // Something broke + cerr << "WriteEngine::FileOp::oid2DirName() error. rc=" << rc << endl; + } } break; @@ -4577,7 +4665,6 @@ int processCommand(string* arguments) cout.width(20); cout << "IP Address"; cout.width(14); - cout << "Status"; if ( AmazonElasticIPCount > 0 ) { cout.width(20); @@ -4656,16 +4743,6 @@ int processCommand(string* arguments) cout << ipAddr; cout.width(14); - try { - oam.getNICStatus(hostname, state); - - printState(state, " "); - } - catch (exception& e) - { - cout << INITIALSTATE; - } - if ( nicID == "1" && AmazonElasticIPCount > 0 ) { int id = 1; @@ -4736,65 +4813,6 @@ int processCommand(string* arguments) cout << endl; - // get and all display Ext Devices Name config parameters - - try - { - SystemExtDeviceConfig systemextdeviceconfig; - oam.getSystemConfig(systemextdeviceconfig); - - if ( systemextdeviceconfig.Count == 0 ) - break; - - cout << endl << "External Device Configuration" << endl << endl; - - cout.setf(ios::left); - cout.width(30); - cout << "Device Name"; - cout.width(20); - cout << "IP Address"; - cout.width(10); - cout << "Status"; - cout << endl; - cout.width(30); - cout << "---------------------"; - cout.width(20); - cout << "---------------"; - cout.width(12); - cout << "------------"; - cout << endl; - - for ( unsigned int i = 0 ; i < systemextdeviceconfig.Count ; i++ ) - { - cout.setf(ios::left); - cout.width(30); - cout << systemextdeviceconfig.extdeviceconfig[i].Name; - cout.width(20); - cout << systemextdeviceconfig.extdeviceconfig[i].IPAddr; - cout.width(12); - - int state; - try { - oam.getExtDeviceStatus(systemextdeviceconfig.extdeviceconfig[i].Name, state); - - printState(state, " "); - } - catch (exception& e) - { - cout << INITIALSTATE; - } - cout << endl; - } - - cout << endl; - } - catch (exception& e) - { - cout << endl << "**** getextdeviceconfig Failed = " << e.what() << endl; - } - - cout << endl; - break; } diff --git a/oamapps/postConfigure/helpers.cpp b/oamapps/postConfigure/helpers.cpp index dd3ff7e2e..a0a2d5825 100644 --- a/oamapps/postConfigure/helpers.cpp +++ b/oamapps/postConfigure/helpers.cpp @@ -114,8 +114,10 @@ void dbrmDirCheck() dbrmroot = sysConfig->getConfig(SystemSection, "DBRMRoot"); dbrmrootPrev = sysConfigPrev->getConfig(SystemSection, "DBRMRoot"); } - catch(...) - {} + catch(const std::exception &exc) + { + std::cerr << exc.what() << std::endl; + } if ( dbrmrootPrev.empty() ) return; @@ -259,8 +261,10 @@ void mysqlSetup() try { oam.actionMysqlCalpont(MYSQL_START); } - catch(...) - {} + catch(const std::exception &exc) + { + std::cerr << exc.what() << std::endl; + } sleep(2); for (;;) @@ -289,7 +293,10 @@ void mysqlSetup() try { oam.setSystemConfig("MySQLPasswordConfig", "y"); } - catch(...) {} + catch(const std::exception &exc) + { + std::cerr << exc.what() << std::endl; + } } else { @@ -303,8 +310,10 @@ void mysqlSetup() try { oam.actionMysqlCalpont(MYSQL_STOP); } - catch(...) - {} + catch(const std::exception &exc) + { + std::cerr << exc.what() << std::endl; + } unlink("/tmp/idbmysql.log"); break; } @@ -344,8 +353,10 @@ int sendUpgradeRequest(int IserverTypeInstall, bool pmwithum) if ( procstat.ProcessOpState == oam::ACTIVE) break; } - catch (exception& ex) - {} + catch(const std::exception &exc) + { + std::cerr << exc.what() << std::endl; + } } SystemModuleTypeConfig systemmoduletypeconfig; @@ -353,8 +364,10 @@ int sendUpgradeRequest(int IserverTypeInstall, bool pmwithum) try{ oam.getSystemConfig(systemmoduletypeconfig); } - catch (exception& ex) - {} + catch(const std::exception &exc) + { + std::cerr << exc.what() << std::endl; + } ByteStream msg; ByteStream::byte requestID = RUNUPGRADE; @@ -393,8 +406,10 @@ int sendUpgradeRequest(int IserverTypeInstall, bool pmwithum) } } } - catch (exception& ex) - {} + catch(const std::exception &exc) + { + std::cerr << exc.what() << std::endl; + } } } } @@ -417,8 +432,10 @@ int sendReplicationRequest(int IserverTypeInstall, std::string password, std::st try{ oam.getSystemConfig(systemmoduletypeconfig); } - catch (exception& ex) - {} + catch(const std::exception &exc) + { + std::cerr << exc.what() << std::endl; + } //get Primary (Master) UM string masterModule = oam::UnassignedName; @@ -474,7 +491,7 @@ int sendReplicationRequest(int IserverTypeInstall, std::string password, std::st returnStatus = sendMsgProcMon( (*pt).DeviceName, msg1, requestID, 600 ); if ( returnStatus != API_SUCCESS) { - cout << "ERROR: Error return in running the MariaDB Columnstore Master DB Distribute, check /tmp/master-dist.logs on " << masterModule << endl; + cout << endl << "ERROR: Error return in running the MariaDB Columnstore Master DB Distribute, check /tmp/master-dist*.logs on " << masterModule << endl; return returnStatus; } @@ -486,7 +503,7 @@ int sendReplicationRequest(int IserverTypeInstall, std::string password, std::st returnStatus = sendMsgProcMon( (*pt).DeviceName, msg, requestID, 30 ); if ( returnStatus != API_SUCCESS) { - cout << "ERROR: Error return in running the MariaDB Columnstore Master replication, check /tmp/master-rep*.logs on " << masterModule << endl; + cout << endl << "ERROR: Error return in running the MariaDB Columnstore Master replication, check /tmp/master-rep*.logs on " << masterModule << endl; return returnStatus; } @@ -517,7 +534,7 @@ int sendReplicationRequest(int IserverTypeInstall, std::string password, std::st returnStatus = sendMsgProcMon( (*pt).DeviceName, msg, requestID, 30 ); if ( returnStatus != API_SUCCESS) { - cout << "ERROR: Error return in running the MariaDB Columnstore Slave replication, check /tmp/slave-rep*.logs on " << (*pt).DeviceName << endl; + cout << endl << "ERROR: Error return in running the MariaDB Columnstore Slave replication, check /tmp/slave-rep*.logs on " << (*pt).DeviceName << endl; return returnStatus; } @@ -526,12 +543,14 @@ int sendReplicationRequest(int IserverTypeInstall, std::string password, std::st } else { - cout << "ERROR: Module not Active, replication not done on " << (*pt).DeviceName << endl; + cout << endl << "ERROR: Module not Active, replication not done on " << (*pt).DeviceName << endl; pt++; } } - catch (exception& ex) - {} + catch(const std::exception &exc) + { + std::cerr << exc.what() << std::endl; + } } else pt++; @@ -593,7 +612,9 @@ int sendMsgProcMon( std::string module, ByteStream msg, int requestID, int timeo try { receivedMSG = mqRequest.read(&ts); } - catch (...) { + catch (const std::exception &exc) + { + std::cerr << exc.what() << std::endl; return returnStatus; } @@ -630,8 +651,10 @@ int sendMsgProcMon( std::string module, ByteStream msg, int requestID, int timeo mqRequest.shutdown(); } - catch (exception& ex) - {} + catch(const std::exception &exc) + { + std::cerr << exc.what() << std::endl; + } return returnStatus; } @@ -651,8 +674,10 @@ void checkFilesPerPartion(int DBRootCount, Config* sysConfig) try { dbRoot = sysConfig->getConfig(SystemSection, "DBRoot1"); } - catch(...) - {} + catch(const std::exception &exc) + { + std::cerr << exc.what() << std::endl; + } dbRoot = dbRoot + "/000.dir"; @@ -661,8 +686,10 @@ void checkFilesPerPartion(int DBRootCount, Config* sysConfig) string tmp = sysConfig->getConfig("ExtentMap", "FilesPerColumnPartition"); FilesPerColumnPartition = atoi(tmp.c_str()); } - catch(...) - {} + catch(const std::exception &exc) + { + std::cerr << exc.what() << std::endl; + } if ( fmod(FilesPerColumnPartition , (float) DBRootCount) != 0 ) { ifstream oldFile (dbRoot.c_str()); @@ -738,8 +765,10 @@ void checkMysqlPort( std::string& mysqlPort, Config* sysConfig ) try { sysConfig->setConfig("Installation", "MySQLPort", mysqlPort); } - catch(...) - {} + catch(const std::exception &exc) + { + std::cerr << exc.what() << std::endl; + } if ( !writeConfig(sysConfig) ) { cout << "ERROR: Failed trying to update MariDB Columnstore System Configuration file" << endl; @@ -860,8 +889,10 @@ void checkSystemMySQLPort(std::string& mysqlPort, Config* sysConfig, std::string try { sysConfig->setConfig("Installation", "MySQLPort", mysqlPort); } - catch(...) - {} + catch(const std::exception &exc) + { + std::cerr << exc.what() << std::endl; + } if ( !writeConfig(sysConfig) ) { cout << "ERROR: Failed trying to update MariDB Columnstore System Configuration file" << endl; @@ -889,8 +920,10 @@ bool writeConfig( Config* sysConfig ) sysConfig->write(); return true; } - catch(...) - {} + catch(const std::exception &exc) + { + std::cerr << exc.what() << std::endl; + } } return false; diff --git a/oamapps/postConfigure/installer.cpp b/oamapps/postConfigure/installer.cpp index 418830649..cc567ce55 100644 --- a/oamapps/postConfigure/installer.cpp +++ b/oamapps/postConfigure/installer.cpp @@ -289,7 +289,7 @@ int main(int argc, char *argv[]) value = "8G"; } - cout << " Setting 'TotalUmMemory' to " << percent << "% of total memory (Combined Server Install maximum value is 16G). Value set to " << value << endl; + cout << " Setting 'TotalUmMemory' to " << percent << "% of total memory. Value set to " << value << endl; try { sysConfig->setConfig("HashJoin", "TotalUmMemory", value); diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index c8353d478..36cf9c762 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -209,6 +209,7 @@ int main(int argc, char *argv[]) noPrompting = false; string password; string cmd; + bool disableAmazon = false; // struct sysinfo myinfo; // hidden options @@ -276,15 +277,17 @@ int main(int argc, char *argv[]) cout << " Enter one of the options within [], if available, or" << endl; cout << " Enter a new value" << endl << endl; cout << endl; - cout << "Usage: postConfigure [-h][-c][-u][-p][-mp][-s][-port]" << endl; + cout << "Usage: postConfigure [-h][-c][-u][-p][-mp][-s][-port][-i][-da]" << endl; cout << " -h Help" << endl; cout << " -c Config File to use to extract configuration data, default is Columnstore.xml.rpmsave" << endl; cout << " -u Upgrade, Install using the Config File from -c, default to Columnstore.xml.rpmsave" << endl; - cout << " If ssh-keys aren't setup, you should provide passwords as command line arguments" << endl; + cout << " If ssh-keys aren't setup, you should provide passwords as command line arguments" << endl; cout << " -p Unix Password, used with no-prompting option" << endl; cout << " -mp MariaDB Columnstore Password" << endl; cout << " -s Single Threaded Remote Install" << endl; cout << " -port MariaDB Columnstore Port Address" << endl; + cout << " -i Non-root Install directory, Only use for non-root installs" << endl; + cout << " -da Disable Amazon functionality, install using Stardard Hostnames and IP Addresses" << endl; exit (0); } else if( string("-s") == argv[i] ) @@ -336,18 +339,6 @@ int main(int argc, char *argv[]) // for backward compatibility else if( string("-n") == argv[i] ) noPrompting = true; - else if( string("-i") == argv[i] ) { - i++; - if (i >= argc ) { - cout << " ERROR: install dir not provided" << endl; - exit (1); - } - installDir = argv[i]; - if ( installDir.find("-") != string::npos ) { - cout << " ERROR: Valid install dir not provided" << endl; - exit (1); - } - } else if( string("-port") == argv[i] ) { i++; if (i >= argc ) { @@ -361,10 +352,20 @@ int main(int argc, char *argv[]) exit (1); } } + else if( string("-da") == argv[i] ) + disableAmazon = true; + else if( string("-i") == argv[i] ) { + i++; + if (i >= argc ) { + cout << " ERROR: Path not provided" << endl; + exit (1); + } + installDir = argv[i]; + } else { cout << " ERROR: Invalid Argument = " << argv[i] << endl; - cout << " Usage: postConfigure [-h][-c][-u][-p][-mp][-s][-port]" << endl; + cout << " Usage: postConfigure [-h][-c][-u][-p][-mp][-s][-port][-i][-da]" << endl; exit (1); } } @@ -814,31 +815,28 @@ int main(int argc, char *argv[]) //amazon install setup check bool amazonInstall = false; - if (rootUser) + string amazonSubNet = oam::UnassignedName; + string cloud = oam::UnassignedName; + if (!disableAmazon) + { system("ec2-version > /tmp/amazon.log 2>&1"); - else - system("sudo ec2-version > /tmp/amazon.log 2>&1"); - ifstream in("/tmp/amazon.log"); + ifstream in("/tmp/amazon.log"); - in.seekg(0, std::ios::end); - int size = in.tellg(); - if ( size == 0 || oam.checkLogStatus("/tmp/amazon.log", "not found")) - // not running on amazon with ec2-api-tools - amazonInstall = false; - else - if ( size == 0 || oam.checkLogStatus("/tmp/amazon.log", "not installed")) + in.seekg(0, std::ios::end); + int size = in.tellg(); + if ( size == 0 || oam.checkLogStatus("/tmp/amazon.log", "not found")) + // not running on amazon with ec2-api-tools amazonInstall = false; else - amazonInstall = true; - - string amazonSubNet = oam::UnassignedName; + if ( size == 0 || oam.checkLogStatus("/tmp/amazon.log", "not installed")) + amazonInstall = false; + else + amazonInstall = true; + } if ( amazonInstall ) { - string cloud = oam::UnassignedName; - string option = "1"; - try { cloud = sysConfig->getConfig(InstallSection, "Cloud"); } @@ -847,69 +845,30 @@ int main(int argc, char *argv[]) cloud = oam::UnassignedName; } - if ( cloud == oam::UnassignedName ) - option = "2"; - - cout << "===== Amazon EC2-API-TOOLS Instance Install =====" << endl << endl; - cout << "You have 2 install options: " << endl << endl; - cout << "1. Utilizing the Amazon IDs for instances and volumes which allows for features like" << endl; - cout << "automaticly launching instances and EBS volumes when configuring and system expansion." << endl; - cout << "This option is recommended and would be use if you are setting up a MariaDB Columnstore system." << endl << endl; - cout << "2. Using standard hardware IDs for hostnames, IP Addresses, and Storage Devices." << endl; - cout << "Using this option, you would need to pre-create the Instances and the EBS storages" << endl; - cout << "and then provide the hostnames/IP-Addresses during the configuration and system expansion" << endl; - cout << "commands. This option would be used when you are installing on a existing system." << endl << endl; + cout << endl << "Amazon EC2 Install, these files will need to be installed on the local instance:" << endl << endl; + cout << " 1. File containing the Amazon Access Key" << endl; + cout << " 2. File containing the Amazon Secret Key" << endl << endl; while(true) { - prompt = "Select Install Option [1,2] (" + option + ") > "; + string ready = "y"; + prompt = "Are these files installed and ready to continue [y,n] (y) > "; pcommand = callReadline(prompt.c_str()); - if (pcommand) { - if (strlen(pcommand) > 0) option = pcommand; + if (pcommand) { + if (strlen(pcommand) > 0) ready = pcommand; callFree(pcommand); - } - - if (option == "2") - { - amazonInstall = false; - break; - } - else - { - if ( option != "1" ) + if (ready == "n") { + cout << endl << "Please Install these files and re-run postConfigure. exiting..." << endl; + exit(0); + } + + if ( ready != "y" ) { - cout << "Invalid Entry, please enter '1' or '2'" << endl; + cout << "Invalid Entry, please enter 'y' for yes or 'n' for no" << endl; if ( noPrompting ) exit(1); - continue; } } - cout << endl << "To use the EC2-api-tools, these files will need to be installed on" << endl; - cout << "on the local instance:" << endl << endl; - cout << " 1. File containing the Amazon Access Key" << endl; - cout << " 2. File containing the Amazon Secret Key" << endl << endl; - - while(true) { - string ready = "y"; - prompt = "Are these files installed and ready to continue [y,n] (y) > "; - pcommand = callReadline(prompt.c_str()); - if (pcommand) { - if (strlen(pcommand) > 0) ready = pcommand; - callFree(pcommand); - if (ready == "n") { - cout << endl << "Please Install these files and re-run postConfigure. exiting..." << endl; - exit(0); - } - } - - if ( ready == "y" ) - break; - - cout << "Invalid Entry, please enter 'y' for yes or 'n' for no" << endl; - if ( noPrompting ) - exit(1); - } - try { AmazonAccessKey = sysConfig->getConfig(InstallSection, "AmazonAccessKey"); AmazonSecretKey = sysConfig->getConfig(InstallSection, "AmazonSecretKey"); @@ -966,30 +925,30 @@ int main(int argc, char *argv[]) if( !copyKeyfiles() ) cout << "copyKeyfiles error" << endl; - try { - AmazonRegion = sysConfig->getConfig(InstallSection, "AmazonRegion"); - } - catch(...) - {} - - cout << endl; - - prompt = "Enter Amazon Region you are running in (" + AmazonRegion + ") > "; - pcommand = callReadline(prompt.c_str()); - if (pcommand) { - if (strlen(pcommand) > 0) AmazonRegion = pcommand; - callFree(pcommand); - } - - try { - sysConfig->setConfig(InstallSection, "AmazonRegion", AmazonRegion); - } - catch(...) - {} - break; } + try { + AmazonRegion = sysConfig->getConfig(InstallSection, "AmazonRegion"); + } + catch(...) + {} + + cout << endl; + + prompt = "Enter Amazon Region you are running in (" + AmazonRegion + ") > "; + pcommand = callReadline(prompt.c_str()); + if (pcommand) { + if (strlen(pcommand) > 0) AmazonRegion = pcommand; + callFree(pcommand); + } + + try { + sysConfig->setConfig(InstallSection, "AmazonRegion", AmazonRegion); + } + catch(...) + {} + if ( !writeConfig(sysConfig) ) { cout << "ERROR: Failed trying to update MariaDB Columnstore System Configuration file" << endl; exit(1); @@ -997,54 +956,51 @@ int main(int argc, char *argv[]) sleep(1); - if ( amazonInstall ) - { - //get subnetID - try { - amazonSubNet = sysConfig->getConfig(InstallSection, "AmazonSubNetID"); - } - catch(...) - {} + //get subnetID + try { + amazonSubNet = sysConfig->getConfig(InstallSection, "AmazonSubNetID"); + } + catch(...) + {} - if ( amazonSubNet == oam::UnassignedName ) - { - //check if this is a vpc system by checking for subnet setup - amazonSubNet = oam.getEC2LocalInstanceSubnet(); - // cout << "amazonSubNet = " << amazonSubNet << endl; - if ( amazonSubNet == "failed" || amazonSubNet == "" ) - { - amazonSubNet = oam::UnassignedName; - cloud = "amazon-ec2"; - } - else - { - cloud = "amazon-vpc"; - } + if ( amazonSubNet == oam::UnassignedName ) + { + //check if this is a vpc system by checking for subnet setup + amazonSubNet = oam.getEC2LocalInstanceSubnet(); + // cout << "amazonSubNet = " << amazonSubNet << endl; + if ( amazonSubNet == "failed" || amazonSubNet == "" ) + { + amazonSubNet = oam::UnassignedName; + cloud = "amazon-ec2"; + } + else + { + cloud = "amazon-vpc"; + } - //set subnetID - try { - sysConfig->setConfig(InstallSection, "AmazonSubNetID", amazonSubNet); - } - catch(...) - {} - } - else - cloud = "amazon-vpc"; - - try { - sysConfig->setConfig(InstallSection, "Cloud", cloud); - } - catch(...) - {} + //set subnetID + try { + sysConfig->setConfig(InstallSection, "AmazonSubNetID", amazonSubNet); + } + catch(...) + {} } else - { - try { - sysConfig->setConfig(InstallSection, "Cloud", oam::UnassignedName); - } - catch(...) - {} + cloud = "amazon-vpc"; + + try { + sysConfig->setConfig(InstallSection, "Cloud", cloud); } + catch(...) + {} + } + else + { + try { + sysConfig->setConfig(InstallSection, "Cloud", oam::UnassignedName); + } + catch(...) + {} } if ( pmwithum ) @@ -1083,7 +1039,7 @@ int main(int argc, char *argv[]) catch (exception& e) {} //get Parent OAM Module Name - string parentOAMModuleName = "pm1"; + parentOAMModuleName = "pm1"; if ( localModuleName != parentOAMModuleName ) { @@ -2754,6 +2710,7 @@ int main(int argc, char *argv[]) /* create a thread_data_t argument array */ thread_data_t thr_data[childmodulelist.size()]; + string install = "y"; if ( IserverTypeInstall != oam::INSTALL_COMBINE_DM_UM_PM || pmNumber > 1 ) { // @@ -2761,7 +2718,6 @@ int main(int argc, char *argv[]) // cout << endl << "===== System Installation =====" << endl << endl; - string install = "y"; cout << "System Configuration is complete, System Installation is the next step." << endl; while(true) @@ -2826,13 +2782,22 @@ int main(int argc, char *argv[]) } if ( EEPackageType == "rpm" ) - cout << "Performing an MariaDB Columnstore System install using RPM packages located in the " + HOME + " directory." << endl; + { + cout << "Performing an MariaDB Columnstore System install using RPM packages" << endl; + cout << " located in the " + HOME + " directory." << endl; + } else { if ( EEPackageType == "binary" ) - cout << "Performing an MariaDB Columnstore System install using a Binary package located in the " + HOME + " directory." << endl; + { + cout << "Performing an MariaDB Columnstore System install using a Binary package" << endl; + cout << "located in the " + HOME + " directory." << endl; + } else - cout << "Performing an MariaDB Columnstore System install using using DEB packages located in the " + HOME + " directory." << endl; + { + cout << "Performing an MariaDB Columnstore System install using using DEB packages" << endl; + cout << "located in the " + HOME + " directory." << endl; + } } //Write out Updated System Configuration File @@ -2936,8 +2901,9 @@ int main(int argc, char *argv[]) break; } - if ( strcmp(pass1, "exit") == 0 ) + if ( pass1 == "exit") exit(0); + string p1 = pass1; pass2=getpass("Confirm password > "); string p2 = pass2; @@ -3397,16 +3363,19 @@ int main(int argc, char *argv[]) string remoteModuleName = (*list1).moduleName; string remoteModuleIP = (*list1).moduleIP; string remoteHostName = (*list1).hostName; - + //run remote command script cout << endl << "----- Starting MariaDB Columnstore on '" + remoteModuleName + "' -----" << endl << endl; - cmd = installDir + "/bin/remote_scp_put.sh " + remoteModuleIP + " " + installDir + "/etc/Columnstore.xml " + installDir + "/etc/. > /dev/null 2>&1"; + if ( install == "n" ) + { // didnt do a full install, push the config file + cmd = installDir + "/bin/remote_scp_put.sh " + remoteModuleIP + " " + installDir + "/etc/Columnstore.xml > /dev/null 2>&1"; + system(cmd.c_str()); + } + + cmd = installDir + "/bin/remote_command.sh " + remoteModuleIP + " " + password + " '" + installDir + "/bin/columnstore restart' 0"; int rtnCode = system(cmd.c_str()); - cmd = installDir + "/bin/remote_command.sh " + remoteModuleIP + " " + password + - " '" + installDir + "/bin/columnstore restart' 0"; - rtnCode = system(cmd.c_str()); if (WEXITSTATUS(rtnCode) != 0) cout << "Error with running remote_command.sh" << endl; else @@ -4990,7 +4959,12 @@ void setSystemName() */ bool copyFstab(string moduleName) { - string cmd = "/bin/cp -f /etc/fstab " + installDir + "/local/etc/" + moduleName + "/. > /dev/null 2>&1"; + string cmd; + if ( rootUser) + cmd = "/bin/cp -f /etc/fstab " + installDir + "/local/etc/" + moduleName + "/. > /dev/null 2>&1"; + else + cmd = "/sudo bin/cp -f /etc/fstab " + installDir + "/local/etc/" + moduleName + "/. > /dev/null 2>&1"; + system(cmd.c_str()); return true; @@ -5042,15 +5016,6 @@ bool updateBash() ifstream newFile (fileName.c_str()); - if (!rootUser) - { - string cmd = "echo export columnstore_INSTALL_DIR=" + installDir + " >> " + fileName; - system(cmd.c_str()); - - cmd = "echo export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$columnstore_INSTALL_DIR/lib:$columnstore_INSTALL_DIR/mysql/lib/mysql >> " + fileName; - system(cmd.c_str()); - } - if ( hdfs ) { string cmd = "echo . " + installDir + "/bin/" + DataFileEnvFile + " >> " + fileName; diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index c56ebfb94..e36827c16 100644 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -4471,7 +4471,7 @@ int ProcessManager::addModule(oam::DeviceNetworkList devicenetworklist, std::str if ( packageType == "deb" ) separator = "_"; //mariadb - calpontPackage = homedir + "/mariadb-columnstore-*" + separator + systemsoftware.Version + "-" + systemsoftware.Release + "*." + packageType; + calpontPackage = homedir + "/mariadb-columnstore*" + separator + systemsoftware.Version + "-" + systemsoftware.Release + "*." + packageType; mysqlPackage = homedir + "/mariadb-columnstore-storage-engine" + separator + systemsoftware.Version + "-" + systemsoftware.Release + "*." + packageType; mysqldPackage = homedir + "/mariadb-columnstore-mysql" + separator + systemsoftware.Version + "-" + systemsoftware.Release + "*." + packageType; calpontPackage1 = homedir + "/mariadb-columnstore-libs" + separator + systemsoftware.Version + "-" + systemsoftware.Release + "*." + packageType; diff --git a/utils/idbdatafile/IDBFileSystem.h b/utils/idbdatafile/IDBFileSystem.h index 2b5c0afbf..d0cc55613 100644 --- a/utils/idbdatafile/IDBFileSystem.h +++ b/utils/idbdatafile/IDBFileSystem.h @@ -84,6 +84,13 @@ public: */ virtual off64_t size(const char* path) const = 0; + /** + * compressedSize() returns the decompressed size of the file + * speicified by path. + * Returns the size on success, -1 on error + */ + virtual off64_t compressedSize(const char* path) const = 0; + /** * exists() checks for the existence of a particular path. * Returns true if exists, false otherwise. diff --git a/utils/idbdatafile/IDBPolicy.h b/utils/idbdatafile/IDBPolicy.h index 81891104c..433bde726 100644 --- a/utils/idbdatafile/IDBPolicy.h +++ b/utils/idbdatafile/IDBPolicy.h @@ -115,6 +115,7 @@ public: */ static int mkdir(const char *pathname); static off64_t size(const char* path); + static off64_t compressedSize(const char* path); static int remove(const char *pathname); static int rename(const char *oldpath, const char *newpath); static bool exists(const char* pathname); @@ -175,6 +176,12 @@ off64_t IDBPolicy::size(const char* path) return IDBPolicy::getFs( path ).size( path ); } +inline +off64_t IDBPolicy::compressedSize(const char* path) +{ + return IDBPolicy::getFs( path ).compressedSize( path ); +} + inline int IDBPolicy::remove(const char *pathname) { diff --git a/utils/idbdatafile/PosixFileSystem.cpp b/utils/idbdatafile/PosixFileSystem.cpp index 38822d2fa..01d542086 100644 --- a/utils/idbdatafile/PosixFileSystem.cpp +++ b/utils/idbdatafile/PosixFileSystem.cpp @@ -17,6 +17,7 @@ #include "PosixFileSystem.h" #include "IDBLogger.h" +#include "idbcompress.h" #include #include @@ -128,6 +129,103 @@ off64_t PosixFileSystem::size(const char* path) const return ret; } +size_t readFillBuffer( + idbdatafile::IDBDataFile* pFile, + char* buffer, + size_t bytesReq) +{ + char* pBuf = buffer; + ssize_t nBytes; + size_t bytesToRead = bytesReq; + size_t totalBytesRead = 0; + + while (1) + { + nBytes = pFile->read(pBuf, bytesToRead); + if (nBytes > 0) + totalBytesRead += nBytes; + else + break; + + if ((size_t)nBytes == bytesToRead) + break; + + pBuf += nBytes; + bytesToRead = bytesToRead - (size_t)nBytes; + } + + return totalBytesRead; +} + +off64_t PosixFileSystem::compressedSize(const char *path) const +{ + IDBDataFile *pFile = NULL; + size_t nBytes; + off64_t dataSize = 0; + + try + { + pFile = IDBDataFile::open(IDBDataFile::BUFFERED, path, "r", 0); + + if (!pFile) + { + return -1; + } + + compress::IDBCompressInterface decompressor; + + char hdr1[compress::IDBCompressInterface::HDR_BUF_LEN]; + nBytes = readFillBuffer( pFile,hdr1,compress::IDBCompressInterface::HDR_BUF_LEN); + if ( nBytes != compress::IDBCompressInterface::HDR_BUF_LEN ) + { + delete pFile; + return -1; + } + + // Verify we are a compressed file + if (decompressor.verifyHdr(hdr1) < 0) + { + delete pFile; + return -1; + } + + int64_t ptrSecSize = decompressor.getHdrSize(hdr1) - compress::IDBCompressInterface::HDR_BUF_LEN; + char* hdr2 = new char[ptrSecSize]; + nBytes = readFillBuffer( pFile,hdr2,ptrSecSize); + if ( (int64_t)nBytes != ptrSecSize ) + { + delete[] hdr2; + delete pFile; + return -1; + } + + compress::CompChunkPtrList chunkPtrs; + int rc = decompressor.getPtrList(hdr2, ptrSecSize, chunkPtrs); + delete[] hdr2; + if (rc != 0) + { + delete pFile; + return -1; + } + + unsigned k = chunkPtrs.size(); + // last header's offset + length will be the data bytes + if (k < 1) + { + delete pFile; + return -1; + } + dataSize = chunkPtrs[k-1].first + chunkPtrs[k-1].second; + delete pFile; + return dataSize; + } + catch (...) + { + delete pFile; + return -1; + } +} + bool PosixFileSystem::exists(const char *pathname) const { boost::filesystem::path dirPath(pathname); diff --git a/utils/idbdatafile/PosixFileSystem.h b/utils/idbdatafile/PosixFileSystem.h index c5c53d162..9c22f805e 100644 --- a/utils/idbdatafile/PosixFileSystem.h +++ b/utils/idbdatafile/PosixFileSystem.h @@ -31,6 +31,7 @@ public: /* virtual */ int mkdir(const char *pathname); /* virtual */ off64_t size(const char* path) const; + /* virtual */ off64_t compressedSize(const char *path) const; /* virtual */ int remove(const char *pathname); /* virtual */ int rename(const char *oldpath, const char *newpath); /* virtual */ bool exists(const char* pathname) const; diff --git a/utils/idbhdfs/hdfs-shared/HdfsFileSystem.cpp b/utils/idbhdfs/hdfs-shared/HdfsFileSystem.cpp index 11ef8bee9..a3a874b38 100644 --- a/utils/idbhdfs/hdfs-shared/HdfsFileSystem.cpp +++ b/utils/idbhdfs/hdfs-shared/HdfsFileSystem.cpp @@ -31,6 +31,7 @@ #include "HdfsFsCache.h" #include "IDBLogger.h" +#include "idbcompress.h" #include #include @@ -119,6 +120,104 @@ off64_t HdfsFileSystem::size(const char* path) const return retval; } +size_t readFillBuffer( + idbdatafile::IDBDataFile* pFile, + char* buffer, + size_t bytesReq) +{ + char* pBuf = buffer; + ssize_t nBytes; + size_t bytesToRead = bytesReq; + size_t totalBytesRead = 0; + + while (1) + { + nBytes = pFile->read(pBuf, bytesToRead); + if (nBytes > 0) + totalBytesRead += nBytes; + else + break; + + if ((size_t)nBytes == bytesToRead) + break; + + pBuf += nBytes; + bytesToRead = bytesToRead - (size_t)nBytes; + } + + return totalBytesRead; +} + + +off64_t HdfsFileSystem::compressedSize(const char *path) const +{ + IDBDataFile *pFile = NULL; + size_t nBytes; + off64_t dataSize = 0; + + try + { + pFile = IDBDataFile::open(IDBDataFile::HDFS, path, "r", 0); + + if (!pFile) + { + return -1; + } + + compress::IDBCompressInterface decompressor; + + char hdr1[compress::IDBCompressInterface::HDR_BUF_LEN]; + nBytes = readFillBuffer( pFile,hdr1,compress::IDBCompressInterface::HDR_BUF_LEN); + if ( nBytes != compress::IDBCompressInterface::HDR_BUF_LEN ) + { + delete pFile; + return -1; + } + + // Verify we are a compressed file + if (decompressor.verifyHdr(hdr1) < 0) + { + delete pFile; + return -1; + } + + int64_t ptrSecSize = decompressor.getHdrSize(hdr1) - compress::IDBCompressInterface::HDR_BUF_LEN; + char* hdr2 = new char[ptrSecSize]; + nBytes = readFillBuffer( pFile,hdr2,ptrSecSize); + if ( (int64_t)nBytes != ptrSecSize ) + { + delete[] hdr2; + delete pFile; + return -1; + } + + compress::CompChunkPtrList chunkPtrs; + int rc = decompressor.getPtrList(hdr2, ptrSecSize, chunkPtrs); + delete[] hdr2; + if (rc != 0) + { + delete pFile; + return -1; + } + + unsigned k = chunkPtrs.size(); + // last header's offset + length will be the data bytes + if (k < 1) + { + delete pFile; + return -1; + } + dataSize = chunkPtrs[k-1].first + chunkPtrs[k-1].second; + delete pFile; + return dataSize; + } + catch (...) + { + delete pFile; + return -1; + } +} + bool HdfsFileSystem::exists(const char *pathname) const { int ret = hdfsExists(m_fs,pathname); diff --git a/utils/idbhdfs/hdfs-shared/HdfsFileSystem.h b/utils/idbhdfs/hdfs-shared/HdfsFileSystem.h index 37c8a58c9..f0a80abe1 100644 --- a/utils/idbhdfs/hdfs-shared/HdfsFileSystem.h +++ b/utils/idbhdfs/hdfs-shared/HdfsFileSystem.h @@ -45,6 +45,7 @@ public: /* virtual */ int mkdir(const char *pathname); /* virtual */ off64_t size(const char* path) const; + /* virtual */ off64_t compressedSize(const char* path) const; /* virtual */ int remove(const char *pathname); /* virtual */ int rename(const char *oldpath, const char *newpath); /* virtual */ bool exists(const char* pathname) const; diff --git a/utils/messageqcpp/inetstreamsocket.cpp b/utils/messageqcpp/inetstreamsocket.cpp index b298843a2..57c863aea 100644 --- a/utils/messageqcpp/inetstreamsocket.cpp +++ b/utils/messageqcpp/inetstreamsocket.cpp @@ -436,10 +436,16 @@ const SBS InetStreamSocket::read(const struct ::timespec* timeout, bool* isTimeO uint8_t* msglenp = reinterpret_cast(&msglen); size_t mlread = 0; - if (readToMagic(msecs, isTimeOut, stats) == false) //indicates a timeout or EOF + bool myIsTimeOut = false; + if (readToMagic(msecs, &myIsTimeOut, stats) == false) //indicates a timeout or EOF { - logIoError("InetStreamSocket::read: timeout during readToMagic", 0); - return SBS(new ByteStream(0)); + if (!myIsTimeOut) + logIoError("InetStreamSocket::read: EOF during readToMagic", 0); + if (isTimeOut) + { + *isTimeOut = myIsTimeOut; + } + return SBS(new ByteStream(0)); } //FIXME: This seems like a lot of work to read 4 bytes... diff --git a/writeengine/server/we_ddlcommandproc.cpp b/writeengine/server/we_ddlcommandproc.cpp index b896b25a3..2ccdd08ae 100644 --- a/writeengine/server/we_ddlcommandproc.cpp +++ b/writeengine/server/we_ddlcommandproc.cpp @@ -193,7 +193,7 @@ uint8_t WE_DDLCommandProc::writeSystable(ByteStream& bs, std::string &err) t = time(NULL); gmtime_r(&t, &tmp); - aDay.year = tmp.tm_year; + aDay.year = tmp.tm_year + 1900; aDay.month = tmp.tm_mon+1; aDay.day = tmp.tm_mday;