From 40c3d05d3a41a8d40363bd9f280cc2a7dd0eb080 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Wed, 9 Nov 2016 13:52:24 +0000 Subject: [PATCH 1/8] Fix rpath settings I accidentally used CMAKE_INSTALL_PREFIX which isn't always set for the engine build. This patch uses INSTALL_ENGINE instead --- CMakeLists.txt | 41 ++++++++++++++++++++--------------------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2df76b54b..82a372f7b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -36,26 +36,6 @@ ENDIF(NOT CMAKE_BUILD_TYPE) #set( CMAKE_VERBOSE_MAKEFILE on ) -# use, i.e. don't skip the full RPATH for the build tree -SET(CMAKE_SKIP_BUILD_RPATH FALSE) - -# when building, don't use the install RPATH already -# (but later on when installing) -SET(CMAKE_BUILD_WITH_INSTALL_RPATH FALSE) - -SET(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib") - -# add the automatically determined parts of the RPATH -# which point to directories outside the build tree to the install RPATH -SET(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) - - -# the RPATH to be used when installing, but only if it's not a system directory -LIST(FIND CMAKE_PLATFORM_IMPLICIT_LINK_DIRECTORIES "${CMAKE_INSTALL_PREFIX}/lib" isSystemDir) -IF("${isSystemDir}" STREQUAL "-1") - SET(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib") -ENDIF("${isSystemDir}" STREQUAL "-1") - SET (PACKAGE columnstore) SET (VERSION 1.0.2) SET (PACKAGE_NAME columnstore) @@ -68,6 +48,25 @@ SET (PACKAGE_URL "") SET (ENGINE_SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}) SET (INSTALL_ENGINE "/usr/local/mariadb/columnstore") +# use, i.e. don't skip the full RPATH for the build tree +SET(CMAKE_SKIP_BUILD_RPATH FALSE) + +# when building, don't use the install RPATH already +# (but later on when installing) +SET(CMAKE_BUILD_WITH_INSTALL_RPATH FALSE) + +SET(CMAKE_INSTALL_RPATH "${INSTALL_ENGINE}/lib") + +# add the automatically determined parts of the RPATH +# which point to directories outside the build tree to the install RPATH +SET(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) + +# the RPATH to be used when installing, but only if it's not a system directory +LIST(FIND CMAKE_PLATFORM_IMPLICIT_LINK_DIRECTORIES "${INSTALL_ENGINE}/lib" isSystemDir) +IF("${isSystemDir}" STREQUAL "-1") + SET(CMAKE_INSTALL_RPATH "${INSTALL_ENGINE}/lib") +ENDIF("${isSystemDir}" STREQUAL "-1") + INCLUDE (configureEngine.cmake) CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/build/releasenum.in ${CMAKE_CURRENT_BINARY_DIR}/build/releasenum @ONLY IMMEDIATE) @@ -110,7 +109,7 @@ ENDFOREACH() SET(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -ggdb3 -fno-tree-vectorize -DSAFE_MUTEX -DSAFEMALLOC -DENABLED_DEBUG_SYNC -O0 -Wall -D_DEBUG -DHAVE_CONFIG_H") SET(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -ggdb3 -fno-tree-vectorize -DSAFE_MUTEX -DSAFEMALLOC -DENABLED_DEBUG_SYNC -O0 -Wall -D _DEBUG -DHAVE_CONFIG_H") -SET (ENGINE_LDFLAGS "-Wl,--rpath -Wl,${INSTALL_ENGINE}/lib -Wl,--no-as-needed -Wl,--add-needed") +SET (ENGINE_LDFLAGS "-Wl,--no-as-needed -Wl,--add-needed") FIND_PACKAGE(Boost 1.53.0 REQUIRED COMPONENTS system filesystem thread regex date_time) From 461052d206a3df88fece4db441b65937a3cb116d Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Wed, 9 Nov 2016 14:02:01 +0000 Subject: [PATCH 2/8] Update MariaDB version number --- README | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README b/README index b56720e1e..85722e3c3 100644 --- a/README +++ b/README @@ -1,6 +1,6 @@ This is MariaDB ColumnStore 1.0.4 MariaDB ColumnStore 1.0.4 is the development version of MariaDB ColumnStore. -It is built by porting InfiniDB 4.6.2 on MariaDB 10.1.18 and adding entirely +It is built by porting InfiniDB 4.6.2 on MariaDB 10.1.19 and adding entirely new features not found anywhere else. MariaDB ColumnStore 1.0.4 is an Beta release. This is the first MariaDB diff --git a/README.md b/README.md index 3754dfc2d..da3148b24 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ #MariaDB ColumnStore Storage/Execution engine 1.0.4 MariaDB ColumnStore 1.0.4 is the development version of MariaDB ColumnStore. -It is built by porting InfiniDB 4.6.2 on MariaDB 10.1.18 and adding entirely +It is built by porting InfiniDB 4.6.2 on MariaDB 10.1.19 and adding entirely new features not found anywhere else. #MariaDB ColumnStore 1.0.4 is an Beta release. From 42cd2032657383c4ca6ec82878a6b21602377cb0 Mon Sep 17 00:00:00 2001 From: Ben Thompson Date: Wed, 9 Nov 2016 13:13:42 -0600 Subject: [PATCH 3/8] MCOL-372: Modify condition which message is logged. Before was logging once a second on every timeout. --- utils/messageqcpp/inetstreamsocket.cpp | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/utils/messageqcpp/inetstreamsocket.cpp b/utils/messageqcpp/inetstreamsocket.cpp index b298843a2..57c863aea 100644 --- a/utils/messageqcpp/inetstreamsocket.cpp +++ b/utils/messageqcpp/inetstreamsocket.cpp @@ -436,10 +436,16 @@ const SBS InetStreamSocket::read(const struct ::timespec* timeout, bool* isTimeO uint8_t* msglenp = reinterpret_cast(&msglen); size_t mlread = 0; - if (readToMagic(msecs, isTimeOut, stats) == false) //indicates a timeout or EOF + bool myIsTimeOut = false; + if (readToMagic(msecs, &myIsTimeOut, stats) == false) //indicates a timeout or EOF { - logIoError("InetStreamSocket::read: timeout during readToMagic", 0); - return SBS(new ByteStream(0)); + if (!myIsTimeOut) + logIoError("InetStreamSocket::read: EOF during readToMagic", 0); + if (isTimeOut) + { + *isTimeOut = myIsTimeOut; + } + return SBS(new ByteStream(0)); } //FIXME: This seems like a lot of work to read 4 bytes... From 5105b3281d2baa5174da2fc0f832aa8ef2f564ee Mon Sep 17 00:00:00 2001 From: Ben Thompson Date: Wed, 9 Nov 2016 17:55:15 -0600 Subject: [PATCH 4/8] MCOL-387: Replaced option flags for version and release to not be tied to CPACK_RPM options. Fixed the defaults used in releasenum file. --- CMakeLists.txt | 19 ++++++++++++++++--- build/releasenum.in | 4 ++-- cpackEngineRPM.cmake | 4 ++-- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 82a372f7b..edbe69e93 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -36,15 +36,28 @@ ENDIF(NOT CMAKE_BUILD_TYPE) #set( CMAKE_VERBOSE_MAKEFILE on ) +IF (NOT VERSION) + SET (VERSION 1.0.5) +ENDIF() +IF (NOT RELEASE) + SET (RELEASE 1) +ENDIF() + SET (PACKAGE columnstore) -SET (VERSION 1.0.2) SET (PACKAGE_NAME columnstore) SET (PACKAGE_TARNAME columnstore) -SET (PACKAGE_VERSION 1.0.2) -SET (PACKAGE_STRING columnstore-1.0.2) SET (PACKAGE_BUGREPORT support@mariadb.com) SET (PACKAGE_URL "") +IF (NOT PACKAGE_VERSION) + SET (PACKAGE_VERSION ${VERSION}) +ENDIF() +IF (NOT PACKAGE_RELEASE) + SET (PACKAGE_RELEASE ${RELEASE}) +ENDIF() +SET (PACKAGE_STRING columnstore-${PACKAGE_VERSION}) + + SET (ENGINE_SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}) SET (INSTALL_ENGINE "/usr/local/mariadb/columnstore") diff --git a/build/releasenum.in b/build/releasenum.in index ea20eb5c1..1a05ad69d 100644 --- a/build/releasenum.in +++ b/build/releasenum.in @@ -1,2 +1,2 @@ -version=@CPACK_RPM_PACKAGE_VERSION@ -release=@CPACK_RPM_PACKAGE_RELEASE@ +version=@VERSION@ +release=@RELEASE@ \ No newline at end of file diff --git a/cpackEngineRPM.cmake b/cpackEngineRPM.cmake index 7cee142a8..0290ea722 100644 --- a/cpackEngineRPM.cmake +++ b/cpackEngineRPM.cmake @@ -15,10 +15,10 @@ SET(CPACK_PACKAGE_NAME "mariadb-columnstore") SET(ENGINE_ARCH "x86_64") IF (NOT CPACK_RPM_PACKAGE_VERSION) -SET (CPACK_RPM_PACKAGE_VERSION "1.0.0") +SET (CPACK_RPM_PACKAGE_VERSION ${PACKAGE_VERSION}) ENDIF() IF (NOT CPACK_RPM_PACKAGE_RELEASE) -SET (CPACK_RPM_PACKAGE_RELEASE "0") +SET (CPACK_RPM_PACKAGE_RELEASE ${PACKAGE_RELEASE}) ENDIF() SET(CPACK_RPM_PACKAGE_NAME ${CPACK_PACKAGE_NAME}) From 874596c6cd1050137406c38de968673ae1952824 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Mon, 14 Nov 2016 15:41:25 +0000 Subject: [PATCH 5/8] MCOL-309 Implement Information Schema tables This patch does the following: * Fix a year storage issue in the datestamp for the columnstore tables metadata table (note that any previous tables will still have the incorrect year stored). * Expose the table creation date in CalpontSystemCatalog::getTables() * Add an INFORMATION_SCHEMA table listing the tables in ColumnStore (similar to systable) * Add an INFORMATION_SCHEMA table listing the columns in ColumnStore (similar to syscolumn) * Add an INFORMATION_SCHEMA table listing the extents in ColumnStore (similar to the editem tool but with additional file information) * Modifies the build system and scripts to support the new tables --- cpackEngineRPM.cmake | 9 + dbcon/execplan/calpontsystemcatalog.cpp | 11 ++ dbcon/execplan/calpontsystemcatalog.h | 1 + dbcon/mysql/CMakeLists.txt | 37 +++- dbcon/mysql/install_calpont_mysql.sh | 3 + dbcon/mysql/is_columnstore_columns.cpp | 187 +++++++++++++++++++ dbcon/mysql/is_columnstore_extents.cpp | 222 +++++++++++++++++++++++ dbcon/mysql/is_columnstore_tables.cpp | 113 ++++++++++++ oam/install_scripts/post-mysqld-install | 11 +- writeengine/server/we_ddlcommandproc.cpp | 2 +- 10 files changed, 592 insertions(+), 4 deletions(-) create mode 100644 dbcon/mysql/is_columnstore_columns.cpp create mode 100644 dbcon/mysql/is_columnstore_extents.cpp create mode 100644 dbcon/mysql/is_columnstore_tables.cpp diff --git a/cpackEngineRPM.cmake b/cpackEngineRPM.cmake index 0290ea722..0a222978e 100644 --- a/cpackEngineRPM.cmake +++ b/cpackEngineRPM.cmake @@ -318,6 +318,15 @@ SET(CPACK_RPM_storage-engine_USER_FILELIST "/usr/local/mariadb/columnstore/lib/libudf_mysql.so.1.0.0" "/usr/local/mariadb/columnstore/lib/libudf_mysql.so.1" "/usr/local/mariadb/columnstore/lib/libudf_mysql.so" +"/usr/local/mariadb/columnstore/lib/is_columnstore_columns.so" +"/usr/local/mariadb/columnstore/lib/is_columnstore_columns.so.1" +"/usr/local/mariadb/columnstore/lib/is_columnstore_columns.so.1.0.0" +"/usr/local/mariadb/columnstore/lib/is_columnstore_extents.so" +"/usr/local/mariadb/columnstore/lib/is_columnstore_extents.so.1" +"/usr/local/mariadb/columnstore/lib/is_columnstore_extents.so.1.0.0" +"/usr/local/mariadb/columnstore/lib/is_columnstore_tables.so" +"/usr/local/mariadb/columnstore/lib/is_columnstore_tables.so.1" +"/usr/local/mariadb/columnstore/lib/is_columnstore_tables.so.1.0.0" "/usr/local/mariadb/columnstore/mysql/mysql-Columnstore" "/usr/local/mariadb/columnstore/mysql/install_calpont_mysql.sh" "/usr/local/mariadb/columnstore/mysql/syscatalog_mysql.sql" diff --git a/dbcon/execplan/calpontsystemcatalog.cpp b/dbcon/execplan/calpontsystemcatalog.cpp index ecec4fcf9..635927c7b 100644 --- a/dbcon/execplan/calpontsystemcatalog.cpp +++ b/dbcon/execplan/calpontsystemcatalog.cpp @@ -2599,6 +2599,7 @@ const vector< pair > SimpleColumn *c1 = new SimpleColumn(CALPONT_SCHEMA+"."+SYSTABLE_TABLE+"."+TABLENAME_COL, fSessionID); SimpleColumn *c2 = new SimpleColumn(CALPONT_SCHEMA+"."+SYSTABLE_TABLE+"."+SCHEMA_COL, fSessionID); SimpleColumn *c3 = new SimpleColumn(CALPONT_SCHEMA+"."+SYSTABLE_TABLE+"."+OBJECTID_COL, fSessionID); + SimpleColumn *c4 = new SimpleColumn(CALPONT_SCHEMA+"."+SYSTABLE_TABLE+"."+CREATEDATE_COL, fSessionID); SRCP srcp; srcp.reset(c1); @@ -2607,6 +2608,8 @@ const vector< pair > colMap.insert(CMVT_(CALPONT_SCHEMA+"."+SYSTABLE_TABLE+"."+SCHEMA_COL, srcp)); srcp.reset(c3); colMap.insert(CMVT_(CALPONT_SCHEMA+"."+SYSTABLE_TABLE+"."+OBJECTID_COL, srcp)); + srcp.reset(c4); + colMap.insert(CMVT_(CALPONT_SCHEMA+"."+SYSTABLE_TABLE+"."+CREATEDATE_COL, srcp)); csep.columnMapNonStatic(colMap); srcp.reset(c1->clone()); @@ -2653,6 +2656,14 @@ const vector< pair > tables[i].second.schema = (*it)->GetStringData(i); } } + for (it = sysDataList.begin(); it != sysDataList.end(); it++) + { + if ((*it)->ColumnOID() == c4->oid()) + { + for (int i = 0; i < (*it)->dataCount(); i++) + tables[i].second.create_date = (*it)->GetData(i); + } + } for (it = sysDataList.begin(); it != sysDataList.end(); it++) { diff --git a/dbcon/execplan/calpontsystemcatalog.h b/dbcon/execplan/calpontsystemcatalog.h index 42b721d65..2936b2daa 100644 --- a/dbcon/execplan/calpontsystemcatalog.h +++ b/dbcon/execplan/calpontsystemcatalog.h @@ -431,6 +431,7 @@ public: TableName(const TableAliasName& tan): schema(tan.schema), table(tan.table) {} std::string schema; std::string table; + int64_t create_date; bool operator<(const TableName& rhs) const; bool operator>=(const TableName& rhs) const { return !(*this < rhs); } bool operator==(const TableName& rhs) const diff --git a/dbcon/mysql/CMakeLists.txt b/dbcon/mysql/CMakeLists.txt index 7dde5b208..7d9b012ca 100644 --- a/dbcon/mysql/CMakeLists.txt +++ b/dbcon/mysql/CMakeLists.txt @@ -17,7 +17,7 @@ SET ( libcalmysql_SRCS ha_view.cpp sm.cpp ha_window_function.cpp ha_calpont_partition.cpp - ha_pseudocolumn.cpp ) + ha_pseudocolumn.cpp) add_definitions(-DMYSQL_DYNAMIC_PLUGIN) @@ -29,7 +29,40 @@ target_link_libraries(calmysql ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${NETSNMP_ set_target_properties(calmysql PROPERTIES VERSION 1.0.0 SOVERSION 1) -install(TARGETS calmysql DESTINATION ${ENGINE_LIBDIR} COMPONENT storage-engine) +SET ( is_columnstore_tables_SRCS + is_columnstore_tables.cpp + ) +add_library(is_columnstore_tables SHARED ${is_columnstore_tables_SRCS}) + +target_link_libraries(is_columnstore_tables ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} threadpool) + +# Don't prepend .so file with 'lib' +set_target_properties(is_columnstore_tables PROPERTIES PREFIX "") +set_target_properties(is_columnstore_tables PROPERTIES VERSION 1.0.0 SOVERSION 1) + +SET ( is_columnstore_columns_SRCS + is_columnstore_columns.cpp + ) +add_library(is_columnstore_columns SHARED ${is_columnstore_columns_SRCS}) + +target_link_libraries(is_columnstore_columns ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} threadpool) + +# Don't prepend .so file with 'lib' +set_target_properties(is_columnstore_columns PROPERTIES PREFIX "") +set_target_properties(is_columnstore_columns PROPERTIES VERSION 1.0.0 SOVERSION 1) + +SET ( is_columnstore_extents_SRCS + is_columnstore_extents.cpp + ) +add_library(is_columnstore_extents SHARED ${is_columnstore_extents_SRCS}) + +target_link_libraries(is_columnstore_extents ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} threadpool) + +# Don't prepend .so file with 'lib' +set_target_properties(is_columnstore_extents PROPERTIES PREFIX "") +set_target_properties(is_columnstore_extents PROPERTIES VERSION 1.0.0 SOVERSION 1) + +install(TARGETS calmysql is_columnstore_tables is_columnstore_columns is_columnstore_extents DESTINATION ${ENGINE_LIBDIR} COMPONENT storage-engine) install(FILES syscatalog_mysql.sql dumpcat_mysql.sql calsetuserpriority.sql diff --git a/dbcon/mysql/install_calpont_mysql.sh b/dbcon/mysql/install_calpont_mysql.sh index 5b2b981df..d04bc3487 100755 --- a/dbcon/mysql/install_calpont_mysql.sh +++ b/dbcon/mysql/install_calpont_mysql.sh @@ -30,6 +30,9 @@ df=$installdir/mysql/my.cnf $installdir/mysql/bin/mysql --defaults-file=$df --force --user=root $pwprompt mysql 2>/tmp/mysql_install.log < +#include + +#include +#include "calpontsystemcatalog.h" +#include "dataconvert.h" + + +// Required declaration as it isn't in a MairaDB include +bool schema_table_store_record(THD *thd, TABLE *table); + +ST_FIELD_INFO is_columnstore_columns_fields[] = +{ + {"TABLE_SCHEMA", 64, MYSQL_TYPE_STRING, 0, 0, 0, 0}, + {"TABLE_NAME", 64, MYSQL_TYPE_STRING, 0, 0, 0, 0}, + {"COLUMN_NAME", 64, MYSQL_TYPE_STRING, 0, 0, 0, 0}, + {"OBJECT_ID", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0}, + {"DICTIONARY_OBJECT_ID", 11, MYSQL_TYPE_LONG, 0, MY_I_S_MAYBE_NULL, 0, 0}, + {"LIST_OBJECT_ID", 11, MYSQL_TYPE_LONG, 0, MY_I_S_MAYBE_NULL, 0, 0}, + {"TREE_OBJECT_ID", 11, MYSQL_TYPE_LONG, 0, MY_I_S_MAYBE_NULL, 0, 0}, + {"DATA_TYPE", 64, MYSQL_TYPE_STRING, 0, 0, 0, 0}, + {"COLUMN_LENGTH", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0}, + {"COLUMN_POSITION", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0}, + {"COLUMN_DEFAULT", 0, MYSQL_TYPE_LONG_BLOB, 0, MY_I_S_MAYBE_NULL, 0, 0}, + {"IS_NULLABLE", 1, MYSQL_TYPE_TINY, 0, 0, 0, 0}, + {"NUMERIC_PRECISION", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0}, + {"NUMERIC_SCALE", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0}, + {"IS_AUTOINCREMENT", 1, MYSQL_TYPE_TINY, 0, 0, 0, 0}, + {"COMPRESSION_TYPE", 64, MYSQL_TYPE_STRING, 0, 0, 0, 0}, + {0, 0, MYSQL_TYPE_NULL, 0, 0, 0, 0} + +}; + +static int is_columnstore_columns_fill(THD *thd, TABLE_LIST *tables, COND *cond) +{ + CHARSET_INFO *cs = system_charset_info; + TABLE *table = tables->table; + + boost::shared_ptr systemCatalogPtr = + execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(0); + + const std::vector< std::pair > catalog_tables + = systemCatalogPtr->getTables(); + + for (std::vector >::const_iterator it = catalog_tables.begin(); + it != catalog_tables.end(); ++it) + { + execplan::CalpontSystemCatalog::RIDList column_rid_list = systemCatalogPtr->columnRIDs((*it).second); + for (size_t col_num = 0; col_num < column_rid_list.size(); col_num++) + { + execplan::CalpontSystemCatalog::TableColName tcn = systemCatalogPtr->colName(column_rid_list[col_num].objnum); + execplan::CalpontSystemCatalog::ColType ct = systemCatalogPtr->colType(column_rid_list[col_num].objnum); + + table->field[0]->store(tcn.schema.c_str(), tcn.schema.length(), cs); + table->field[1]->store(tcn.table.c_str(), tcn.table.length(), cs); + table->field[2]->store(tcn.column.c_str(), tcn.column.length(), cs); + table->field[3]->store(column_rid_list[col_num].objnum); + if (ct.ddn.dictOID == std::numeric_limits::min()) + { + table->field[4]->set_null(); + } + else + { + table->field[4]->set_notnull(); + table->field[4]->store(ct.ddn.dictOID); + } + if (ct.ddn.listOID == std::numeric_limits::min()) + { + table->field[5]->set_null(); + } + else + { + table->field[5]->set_notnull(); + table->field[5]->store(ct.ddn.listOID); + } + if (ct.ddn.treeOID == std::numeric_limits::min()) + { + table->field[6]->set_null(); + } + else + { + table->field[6]->set_notnull(); + table->field[6]->store(ct.ddn.treeOID); + } + std::string data_type = execplan::colDataTypeToString(ct.colDataType); + table->field[7]->store(data_type.c_str(), data_type.length(), cs); + table->field[8]->store(ct.colWidth); + table->field[9]->store(ct.colPosition); + if (ct.defaultValue.empty()) + { + table->field[10]->set_null(); + } + else + { + table->field[10]->set_notnull(); + table->field[10]->store(ct.defaultValue.c_str(), ct.defaultValue.length(), cs); + } + table->field[11]->store(ct.autoincrement); + table->field[12]->store(ct.precision); + table->field[13]->store(ct.scale); + if (ct.constraintType != execplan::CalpontSystemCatalog::NOTNULL_CONSTRAINT) + { + table->field[14]->store(true); + } + else + { + table->field[14]->store(false); + } + + std::string compression_type; + switch (ct.compressionType) + { + case 0: + compression_type = "None"; + break; + case 2: + compression_type = "Snappy"; + break; + default: + compression_type = "Unknown"; + break; + } + table->field[15]->store(compression_type.c_str(), compression_type.length(), cs); + if (schema_table_store_record(thd, table)) + return 1; + } + } + + + + return 0; +} + +static int is_columnstore_columns_plugin_init(void *p) +{ + ST_SCHEMA_TABLE *schema = (ST_SCHEMA_TABLE*) p; + schema->fields_info = is_columnstore_columns_fields; + schema->fill_table = is_columnstore_columns_fill; + return 0; +} + +static struct st_mysql_information_schema is_columnstore_columns_plugin_version = +{ MYSQL_INFORMATION_SCHEMA_INTERFACE_VERSION }; + +maria_declare_plugin(is_columnstore_columns_plugin) +{ + MYSQL_INFORMATION_SCHEMA_PLUGIN, + &is_columnstore_columns_plugin_version, + "COLUMNSTORE_COLUMNS", + "MariaDB Corporaton", + "An information schema plugin to list ColumnStore columns", + PLUGIN_LICENSE_GPL, + is_columnstore_columns_plugin_init, + //is_columnstore_tables_plugin_deinit, + NULL, + 0x0100, + NULL, + NULL, + "1.0", + MariaDB_PLUGIN_MATURITY_STABLE +} +maria_declare_plugin_end; + + diff --git a/dbcon/mysql/is_columnstore_extents.cpp b/dbcon/mysql/is_columnstore_extents.cpp new file mode 100644 index 000000000..fa5835888 --- /dev/null +++ b/dbcon/mysql/is_columnstore_extents.cpp @@ -0,0 +1,222 @@ +/* c-basic-offset: 4; tab-width: 4; indent-tabs-mode: nil + * vi: set shiftwidth=4 tabstop=4 expandtab: + * :indentSize=4:tabSize=4:noTabs=true: + * + * Copyright (C) 2016 MariaDB Corporaton + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; version 2 of + * the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301, USA. + */ + +#include "idb_mysql.h" +#include +#include + +#include +#include "dbrm.h" +#include "objectidmanager.h" +#include "we_convertor.h" +#include "we_define.h" +#include "IDBPolicy.h" +#include "we_config.h" + +// Required declaration as it isn't in a MairaDB include +bool schema_table_store_record(THD *thd, TABLE *table); + +ST_FIELD_INFO is_columnstore_extents_fields[] = +{ + {"OBJECT_ID", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0}, // 0 + {"OBJECT_TYPE", 64, MYSQL_TYPE_STRING, 0, 0, 0, 0}, // 1 + {"LOGICAL_BLOCK_START", 19, MYSQL_TYPE_LONGLONG, 0, 0, 0, 0}, // 2 + {"LOGICAL_BLOCK_END", 19, MYSQL_TYPE_LONGLONG, 0, 0, 0, 0}, // 3 + {"MIN_VALUE", 19, MYSQL_TYPE_LONGLONG, 0, MY_I_S_MAYBE_NULL, 0, 0}, // 4 + {"MAX_VALUE", 19, MYSQL_TYPE_LONGLONG, 0, MY_I_S_MAYBE_NULL, 0, 0}, // 5 + {"WIDTH", 1, MYSQL_TYPE_TINY, 0, 0, 0, 0}, // 6 + {"DBROOT", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0}, // 7 + {"PARTITION_ID", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0}, // 8 + {"SEGMENT_ID", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0}, // 9 + {"BLOCK_OFFSET", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0}, // 10 + {"MAX_BLOCKS", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0}, // 11 + {"HIGH_WATER_MARK", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0}, // 12 + {"STATE", 64, MYSQL_TYPE_STRING, 0, 0, 0, 0}, // 13 + {"STATUS", 64, MYSQL_TYPE_STRING, 0, 0, 0, 0}, // 14 + {"DATA_SIZE", 19, MYSQL_TYPE_LONGLONG, 0, 0, 0, 0}, // 15 + {"FILENAME", 1024, MYSQL_TYPE_STRING, 0, 0, 0, 0}, // 16 + {"FILE_SIZE", 19, MYSQL_TYPE_LONGLONG, 0, MY_I_S_MAYBE_NULL, 0, 0}, // 17 + {0, 0, MYSQL_TYPE_NULL, 0, 0, 0, 0} +}; + +static int is_columnstore_extents_fill(THD *thd, TABLE_LIST *tables, COND *cond) +{ + CHARSET_INFO *cs = system_charset_info; + TABLE *table = tables->table; + std::vector entries; + std::vector::iterator iter; + std::vector::iterator end; + char oidDirName[WriteEngine::FILE_NAME_SIZE]; + char fullFileName[WriteEngine::FILE_NAME_SIZE]; + char dbDir[WriteEngine::MAX_DB_DIR_LEVEL][WriteEngine::MAX_DB_DIR_NAME_SIZE]; + + BRM::DBRM *emp = new BRM::DBRM(); + if (!emp || !emp->isDBRMReady()) + { + return 1; + } + + execplan::ObjectIDManager oidm; + BRM::OID_t MaxOID = oidm.size(); + + for(BRM::OID_t oid = 0; oid <= MaxOID; oid++) + { + emp->getExtents(oid, entries, false, false, true); + if (entries.size() == 0) + continue; + + iter = entries.begin(); + end = entries.end(); + while (iter != end) + { + table->field[0]->store(oid); + if (iter->colWid > 0) + { + table->field[1]->store("Column", strlen("Column"), cs); + if (iter->partition.cprange.lo_val == std::numeric_limits::max() || + iter->partition.cprange.lo_val <= (std::numeric_limits::min() + 2)) + { + table->field[4]->set_null(); + } + else + { + table->field[4]->set_notnull(); + table->field[4]->store(iter->partition.cprange.lo_val); + } + if (iter->partition.cprange.hi_val == std::numeric_limits::max() || + iter->partition.cprange.hi_val <= (std::numeric_limits::min() + 2)) + { + table->field[5]->set_null(); + } + else + { + table->field[5]->set_notnull(); + table->field[5]->store(iter->partition.cprange.hi_val); + } + table->field[6]->store(iter->colWid); + + } + else + { + table->field[1]->store("Dictionary", strlen("Dictionary"), cs); + table->field[4]->set_null(); + table->field[5]->set_null(); + table->field[6]->store(8192); + } + table->field[2]->store(iter->range.start); + table->field[3]->store(iter->range.start + (iter->range.size * 1024) - 1); + + table->field[7]->store(iter->dbRoot); + table->field[8]->store(iter->partitionNum); + table->field[9]->store(iter->segmentNum); + table->field[10]->store(iter->blockOffset); + table->field[11]->store(iter->range.size * 1024); + table->field[12]->store(iter->HWM); + + switch (iter->partition.cprange.isValid) + { + case 0: + table->field[13]->store("Invalid", strlen("Invalid"), cs); + break; + case 1: + table->field[13]->store("Updating", strlen("Updating"), cs); + break; + case 2: + table->field[13]->store("Valid", strlen("Valid"), cs); + break; + default: + table->field[13]->store("Unknown", strlen("Unknown"), cs); + break; + } + switch (iter->status) + { + case BRM::EXTENTAVAILABLE: + table->field[14]->store("Available", strlen("Available"), cs); + break; + case BRM::EXTENTUNAVAILABLE: + table->field[14]->store("Unavailable", strlen("Unavailable"), cs); + break; + case BRM::EXTENTOUTOFSERVICE: + table->field[14]->store("Out of service", strlen("Out of service"), cs); + break; + default: + table->field[14]->store("Unknown", strlen("Unknown"), cs); + } + table->field[15]->store((iter->HWM + 1) * 8192); + + WriteEngine::Convertor::oid2FileName(oid, oidDirName, dbDir, iter->partitionNum, iter->segmentNum); + snprintf(fullFileName, WriteEngine::FILE_NAME_SIZE, "%s/%s", WriteEngine::Config::getDBRootByNum(iter->dbRoot).c_str(), oidDirName); + table->field[16]->store(fullFileName, strlen(fullFileName), cs); + + if (idbdatafile::IDBPolicy::exists(fullFileName)) + { + table->field[17]->set_notnull(); + table->field[17]->store(idbdatafile::IDBPolicy::size(fullFileName)); + } + else + { + table->field[17]->set_null(); + } + + if (schema_table_store_record(thd, table)) + return 1; + + iter++; + + } + } + + + return 0; +} + +static int is_columnstore_extents_plugin_init(void *p) +{ + ST_SCHEMA_TABLE *schema = (ST_SCHEMA_TABLE*) p; + schema->fields_info = is_columnstore_extents_fields; + schema->fill_table = is_columnstore_extents_fill; + return 0; +} + +static struct st_mysql_information_schema is_columnstore_extents_plugin_version = +{ MYSQL_INFORMATION_SCHEMA_INTERFACE_VERSION }; + +maria_declare_plugin(is_columnstore_extents_plugin) +{ + MYSQL_INFORMATION_SCHEMA_PLUGIN, + &is_columnstore_extents_plugin_version, + "COLUMNSTORE_EXTENTS", + "MariaDB Corporaton", + "An information schema plugin to list ColumnStore extents", + PLUGIN_LICENSE_GPL, + is_columnstore_extents_plugin_init, + //is_columnstore_extents_plugin_deinit, + NULL, + 0x0100, + NULL, + NULL, + "1.0", + MariaDB_PLUGIN_MATURITY_STABLE +} +maria_declare_plugin_end; + + diff --git a/dbcon/mysql/is_columnstore_tables.cpp b/dbcon/mysql/is_columnstore_tables.cpp new file mode 100644 index 000000000..e38c8d5e7 --- /dev/null +++ b/dbcon/mysql/is_columnstore_tables.cpp @@ -0,0 +1,113 @@ +/* c-basic-offset: 4; tab-width: 4; indent-tabs-mode: nil + * vi: set shiftwidth=4 tabstop=4 expandtab: + * :indentSize=4:tabSize=4:noTabs=true: + * + * Copyright (C) 2016 MariaDB Corporaton + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; version 2 of + * the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301, USA. + */ + +#include "idb_mysql.h" +#include + +#include +#include "calpontsystemcatalog.h" +#include "dataconvert.h" + + +// Required declaration as it isn't in a MairaDB include +bool schema_table_store_record(THD *thd, TABLE *table); + +ST_FIELD_INFO is_columnstore_tables_fields[] = +{ + {"TABLE_SCHEMA", 64, MYSQL_TYPE_STRING, 0, 0, 0, 0}, + {"TABLE_NAME", 64, MYSQL_TYPE_STRING, 0, 0, 0, 0}, + {"OBJECT_ID", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0}, + {"CREATION_DATE", 0, MYSQL_TYPE_DATE, 0, 0, 0, 0}, + {"COLUMN_COUNT", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0}, + {"AUTOINCREMENT", 11, MYSQL_TYPE_LONG, 0, MY_I_S_MAYBE_NULL, 0, 0}, + {0, 0, MYSQL_TYPE_NULL, 0, 0, 0, 0} +}; + +static int is_columnstore_tables_fill(THD *thd, TABLE_LIST *tables, COND *cond) +{ + CHARSET_INFO *cs = system_charset_info; + TABLE *table = tables->table; + + boost::shared_ptr systemCatalogPtr = + execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(0); + + const std::vector< std::pair > catalog_tables + = systemCatalogPtr->getTables(); + + for (std::vector >::const_iterator it = catalog_tables.begin(); + it != catalog_tables.end(); ++it) + { + execplan::CalpontSystemCatalog::TableInfo tb_info = systemCatalogPtr->tableInfo((*it).second); + std::string create_date = dataconvert::DataConvert::dateToString((*it).second.create_date); + table->field[0]->store((*it).second.schema.c_str(), (*it).second.schema.length(), cs); + table->field[1]->store((*it).second.table.c_str(), (*it).second.table.length(), cs); + table->field[2]->store((*it).first); + table->field[3]->store(create_date.c_str(), create_date.length(), cs); + table->field[4]->store(tb_info.numOfCols); + if (tb_info.tablewithautoincr) + { + table->field[5]->set_notnull(); + table->field[5]->store(systemCatalogPtr->nextAutoIncrValue((*it).second)); + } + else + { + table->field[5]->set_null(); + } + table->field[5]->store(tb_info.tablewithautoincr); + if (schema_table_store_record(thd, table)) + return 1; + } + + return 0; +} + +static int is_columnstore_tables_plugin_init(void *p) +{ + ST_SCHEMA_TABLE *schema = (ST_SCHEMA_TABLE*) p; + schema->fields_info = is_columnstore_tables_fields; + schema->fill_table = is_columnstore_tables_fill; + return 0; +} + +static struct st_mysql_information_schema is_columnstore_tables_plugin_version = +{ MYSQL_INFORMATION_SCHEMA_INTERFACE_VERSION }; + +maria_declare_plugin(is_columnstore_tables_plugin) +{ + MYSQL_INFORMATION_SCHEMA_PLUGIN, + &is_columnstore_tables_plugin_version, + "COLUMNSTORE_TABLES", + "MariaDB Corporaton", + "An information schema plugin to list ColumnStore tables", + PLUGIN_LICENSE_GPL, + is_columnstore_tables_plugin_init, + //is_columnstore_tables_plugin_deinit, + NULL, + 0x0100, + NULL, + NULL, + "1.0", + MariaDB_PLUGIN_MATURITY_STABLE +} +maria_declare_plugin_end; + + diff --git a/oam/install_scripts/post-mysqld-install b/oam/install_scripts/post-mysqld-install index 833f5f26c..92ea035fa 100755 --- a/oam/install_scripts/post-mysqld-install +++ b/oam/install_scripts/post-mysqld-install @@ -82,9 +82,15 @@ chown -R $user.$user $installdir/mysql if [ -f $installdir/lib/libcalmysql.so.1.0.0 ]; then libcalmysql=$installdir/lib/libcalmysql.so.1.0.0 libudfsdk=$installdir/lib/libudf_mysql.so.1.0.0 + is_columnstore_tables=$installdir/lib/is_columnstore_tables.so.1.0.0 + is_columnstore_columns=$installdir/lib/is_columnstore_columns.so.1.0.0 + is_columnstore_extents=$installdir/lib/is_columnstore_extents.so.1.0.0 elif [ -f $installdir/lib/libcalmysql.so.1 ]; then libcalmysql=$installdir/lib/libcalmysql.so.1 libudfsdk=$installdir/lib/libudf_mysql.so.1 + is_columnstore_tables=$installdir/lib/is_columnstore_tables.so.1 + is_columnstore_columns=$installdir/lib/is_columnstore_columns.so.1 + is_columnstore_extents=$installdir/lib/is_columnstore_extents.so.1 else libcalmysql= fi @@ -93,8 +99,11 @@ if [ -n "$libcalmysql" ]; then cd $installdir/mysql/lib/plugin ln -sf $libcalmysql libcalmysql.so - ln -sf $libcalmysql libcalmysqlent.so + ln -sf $libcalmysql libcalmysqlent.so ln -sf $libudfsdk libudf_mysql.so + ln -sf $is_columnstore_tables is_columnstore_tables.so + ln -sf $is_columnstore_columns is_columnstore_columns.so + ln -sf $is_columnstore_extents is_columnstore_extents.so fi # cleanup previous install mysql replication files diff --git a/writeengine/server/we_ddlcommandproc.cpp b/writeengine/server/we_ddlcommandproc.cpp index b896b25a3..2ccdd08ae 100644 --- a/writeengine/server/we_ddlcommandproc.cpp +++ b/writeengine/server/we_ddlcommandproc.cpp @@ -193,7 +193,7 @@ uint8_t WE_DDLCommandProc::writeSystable(ByteStream& bs, std::string &err) t = time(NULL); gmtime_r(&t, &tmp); - aDay.year = tmp.tm_year; + aDay.year = tmp.tm_year + 1900; aDay.month = tmp.tm_mon+1; aDay.day = tmp.tm_mday; From cc0f5efa28bbdeb7b67c427d39995da5382fd89e Mon Sep 17 00:00:00 2001 From: Ben Thompson Date: Tue, 15 Nov 2016 11:18:27 -0600 Subject: [PATCH 6/8] MCOL-386: Added some error reporting in catch statements in helpers.cpp --- oamapps/postConfigure/helpers.cpp | 93 +++++++++++++++++++++---------- 1 file changed, 63 insertions(+), 30 deletions(-) diff --git a/oamapps/postConfigure/helpers.cpp b/oamapps/postConfigure/helpers.cpp index dd3ff7e2e..36cca5177 100644 --- a/oamapps/postConfigure/helpers.cpp +++ b/oamapps/postConfigure/helpers.cpp @@ -114,8 +114,10 @@ void dbrmDirCheck() dbrmroot = sysConfig->getConfig(SystemSection, "DBRMRoot"); dbrmrootPrev = sysConfigPrev->getConfig(SystemSection, "DBRMRoot"); } - catch(...) - {} + catch(const std::exception &exc) + { + std::cerr << exc.what() << std::endl; + } if ( dbrmrootPrev.empty() ) return; @@ -259,8 +261,10 @@ void mysqlSetup() try { oam.actionMysqlCalpont(MYSQL_START); } - catch(...) - {} + catch(const std::exception &exc) + { + std::cerr << exc.what() << std::endl; + } sleep(2); for (;;) @@ -289,7 +293,10 @@ void mysqlSetup() try { oam.setSystemConfig("MySQLPasswordConfig", "y"); } - catch(...) {} + catch(const std::exception &exc) + { + std::cerr << exc.what() << std::endl; + } } else { @@ -303,8 +310,10 @@ void mysqlSetup() try { oam.actionMysqlCalpont(MYSQL_STOP); } - catch(...) - {} + catch(const std::exception &exc) + { + std::cerr << exc.what() << std::endl; + } unlink("/tmp/idbmysql.log"); break; } @@ -344,8 +353,10 @@ int sendUpgradeRequest(int IserverTypeInstall, bool pmwithum) if ( procstat.ProcessOpState == oam::ACTIVE) break; } - catch (exception& ex) - {} + catch(const std::exception &exc) + { + std::cerr << exc.what() << std::endl; + } } SystemModuleTypeConfig systemmoduletypeconfig; @@ -353,8 +364,10 @@ int sendUpgradeRequest(int IserverTypeInstall, bool pmwithum) try{ oam.getSystemConfig(systemmoduletypeconfig); } - catch (exception& ex) - {} + catch(const std::exception &exc) + { + std::cerr << exc.what() << std::endl; + } ByteStream msg; ByteStream::byte requestID = RUNUPGRADE; @@ -393,8 +406,10 @@ int sendUpgradeRequest(int IserverTypeInstall, bool pmwithum) } } } - catch (exception& ex) - {} + catch(const std::exception &exc) + { + std::cerr << exc.what() << std::endl; + } } } } @@ -417,8 +432,10 @@ int sendReplicationRequest(int IserverTypeInstall, std::string password, std::st try{ oam.getSystemConfig(systemmoduletypeconfig); } - catch (exception& ex) - {} + catch(const std::exception &exc) + { + std::cerr << exc.what() << std::endl; + } //get Primary (Master) UM string masterModule = oam::UnassignedName; @@ -530,8 +547,10 @@ int sendReplicationRequest(int IserverTypeInstall, std::string password, std::st pt++; } } - catch (exception& ex) - {} + catch(const std::exception &exc) + { + std::cerr << exc.what() << std::endl; + } } else pt++; @@ -593,7 +612,9 @@ int sendMsgProcMon( std::string module, ByteStream msg, int requestID, int timeo try { receivedMSG = mqRequest.read(&ts); } - catch (...) { + catch (const std::exception &exc) + { + std::cerr << exc.what() << std::endl; return returnStatus; } @@ -630,8 +651,10 @@ int sendMsgProcMon( std::string module, ByteStream msg, int requestID, int timeo mqRequest.shutdown(); } - catch (exception& ex) - {} + catch(const std::exception &exc) + { + std::cerr << exc.what() << std::endl; + } return returnStatus; } @@ -651,8 +674,10 @@ void checkFilesPerPartion(int DBRootCount, Config* sysConfig) try { dbRoot = sysConfig->getConfig(SystemSection, "DBRoot1"); } - catch(...) - {} + catch(const std::exception &exc) + { + std::cerr << exc.what() << std::endl; + } dbRoot = dbRoot + "/000.dir"; @@ -661,8 +686,10 @@ void checkFilesPerPartion(int DBRootCount, Config* sysConfig) string tmp = sysConfig->getConfig("ExtentMap", "FilesPerColumnPartition"); FilesPerColumnPartition = atoi(tmp.c_str()); } - catch(...) - {} + catch(const std::exception &exc) + { + std::cerr << exc.what() << std::endl; + } if ( fmod(FilesPerColumnPartition , (float) DBRootCount) != 0 ) { ifstream oldFile (dbRoot.c_str()); @@ -738,8 +765,10 @@ void checkMysqlPort( std::string& mysqlPort, Config* sysConfig ) try { sysConfig->setConfig("Installation", "MySQLPort", mysqlPort); } - catch(...) - {} + catch(const std::exception &exc) + { + std::cerr << exc.what() << std::endl; + } if ( !writeConfig(sysConfig) ) { cout << "ERROR: Failed trying to update MariDB Columnstore System Configuration file" << endl; @@ -860,8 +889,10 @@ void checkSystemMySQLPort(std::string& mysqlPort, Config* sysConfig, std::string try { sysConfig->setConfig("Installation", "MySQLPort", mysqlPort); } - catch(...) - {} + catch(const std::exception &exc) + { + std::cerr << exc.what() << std::endl; + } if ( !writeConfig(sysConfig) ) { cout << "ERROR: Failed trying to update MariDB Columnstore System Configuration file" << endl; @@ -889,8 +920,10 @@ bool writeConfig( Config* sysConfig ) sysConfig->write(); return true; } - catch(...) - {} + catch(const std::exception &exc) + { + std::cerr << exc.what() << std::endl; + } } return false; From a74b03fdad9443f8e95c1f44d166a10d23d90e21 Mon Sep 17 00:00:00 2001 From: David Hill Date: Wed, 16 Nov 2016 23:22:25 +0000 Subject: [PATCH 7/8] mcol-404, non-root install fixes --- oam/install_scripts/columnstore | 20 ++++++----- oam/install_scripts/post-install | 13 ++++++-- oam/install_scripts/pre-uninstall | 44 +++++++++++++++---------- oam/oamcpp/liboamcpp.cpp | 2 +- oamapps/alarmmanager/alarmmanager.cpp | 20 +++-------- oamapps/postConfigure/postConfigure.cpp | 11 +++---- 6 files changed, 58 insertions(+), 52 deletions(-) diff --git a/oam/install_scripts/columnstore b/oam/install_scripts/columnstore index c346e3eb0..315256cd9 100644 --- a/oam/install_scripts/columnstore +++ b/oam/install_scripts/columnstore @@ -57,8 +57,10 @@ if [ "x$has_um" = x ]; then fi user=$USER +sudo=sudo if [ -z "$user" ]; then user=root + sudo=" " fi checkInstallSetup() { @@ -66,7 +68,7 @@ checkInstallSetup() { if [ $InitialInstallFlag != "y" ]; then echo "Please run the postConfigure install script, check the Installation Guide" echo "for additional details" - rm -f /var/lock/subsys/columnstore + sudo rm -f /var/lock/subsys/columnstore exit 1 fi } @@ -79,13 +81,13 @@ start() { exit 0 fi - (mkdir -p /var/lock/subsys && touch /var/lock/subsys/columnstore) >/dev/null 2>&1 + (sudo mkdir -p /var/lock/subsys && touch /var/lock/subsys/columnstore) >/dev/null 2>&1 if [ -x $InstallDir/bin/columnstore.pre-start ]; then $InstallDir/bin/columnstore.pre-start if [ $? -ne 0 ]; then echo "Error running MariaDB Columnstore pre-start script, not starting MariaDB Columnstore" - rm -f /var/lock/subsys/columnstore + sudo rm -f /var/lock/subsys/columnstore exit 1 fi fi @@ -106,21 +108,21 @@ start() { RETVAL=0 echo "Starting MariaDB Columnstore Database Platform" - rm -f /tmp/StopColumnstore + sudo rm -f /tmp/StopColumnstore exec $InstallDir/bin/run.sh $InstallDir/bin/ProcMon > /dev/null 2>&1 & return $RETVAL } stop() { echo "Shutting down MariaDB Columnstore Database Platform" - touch /tmp/StopColumnstore - pkill ProcMon - pkill ProcMgr + sudo touch /tmp/StopColumnstore + sudo pkill ProcMon + sudo pkill ProcMgr sleep 1 $InstallDir/bin/clearShm RETVAL=$? - rm -f /var/lock/subsys/columnstore - fuser -k 8604/tcp > /dev/null 2>&1 + sudo rm -f /var/lock/subsys/columnstore + sudo fuser -k 8604/tcp > /dev/null 2>&1 test -f $InstallDir/mysql/mysql-Columnstore || return $RETVAL $InstallDir/mysql/mysql-Columnstore stop > /dev/null 2>&1 if [ -x $InstallDir/bin/columnstore.post-stop ]; then diff --git a/oam/install_scripts/post-install b/oam/install_scripts/post-install index 38ad26fe6..6e245b31b 100755 --- a/oam/install_scripts/post-install +++ b/oam/install_scripts/post-install @@ -95,7 +95,16 @@ sudo ln -s /usr/lib64/perl5/CORE/libperl.so /usr/lib64/libperl.so >/dev/null 2>& cd / +if [ $user != "root" ]; then + sudo chmod 777 /var + sudo chmod 777 /var/log +fi + +test -d /var/log/mariadb || mkdir /var/log/mariadb >/dev/null 2>&1 +sudo chmod 777 /var/log/mariadb test -d /var/log/mariadb/columnstore || mkdir /var/log/mariadb/columnstore >/dev/null 2>&1 +sudo chmod 777 /var/log/mariadb/columnstore + test -d /var/log/mariadb/columnstore/archive || mkdir /var/log/mariadb/columnstore/archive >/dev/null 2>&1 test -d /var/log/mariadb/columnstore/corefiles || mkdir /var/log/mariadb/columnstore/corefiles >/dev/null 2>&1 test -d /var/log/mariadb/columnstore/trace || mkdir /var/log/mariadb/columnstore/trace >/dev/null 2>&1 @@ -182,9 +191,9 @@ if [ $user = "root" ]; then else sudo $installdir/bin/syslogSetup.sh --installdir=$installdir install > /tmp/syslog_install.log 2>&1 sudo chown $user:$user $installdir/etc/Columnstore.xml - sudo chmod 755 /var/log - sudo chmod -R 755 /var/log/mariadb/columnstore sudo chmod -R 777 /dev/shm + sudo chmod 777 /var/lock/subsys + sudo chmod 777 /tmp sudo rm -f /var/lock/subsys/mysql-Columnstore sudo rm -f /tmp/StopColumnstore diff --git a/oam/install_scripts/pre-uninstall b/oam/install_scripts/pre-uninstall index 464071188..bf3bbc053 100755 --- a/oam/install_scripts/pre-uninstall +++ b/oam/install_scripts/pre-uninstall @@ -38,17 +38,27 @@ $installdir/bin/columnstore stop > /dev/null 2>&1 $installdir/myql/columnstore-Mysql stop > /dev/null 2>&1 # uninstall OS scripts updated by postConfigure -if test -f /root/.bashrc.columnstoreSave ; then - $sudo mv -f /root/.bashrc.columnstoreSave /root/.bashrc > /dev/null 2>&1 +if [ $user = "root" ]; then + if test -f /root/.bashrc.columnstoreSave ; then + mv -f /root/.bashrc.columnstoreSave /root/.bashrc > /dev/null 2>&1 + fi +else + if test -f $HOME/.bashrc.columnstoreSave ; then + mv -f /home/$user/.bashrc.columnstoreSave /home/$user/.bashrc > /dev/null 2>&1 + fi fi + + if test -f /etc/exports.columnstoreSave ; then $sudo mv -f /etc/exports.columnstoreSave /etc/exports > /dev/null 2>&1 else $sudo rm -f /etc/exports > /dev/null 2>&1 fi -if test -f /etc/rc.d/rc.local.columnstoreSave ; then + +if test -f /etc/rc.d/rc.local.columnstoreSave ; then $sudo mv -f /etc/rc.d/rc.local.columnstoreSave /etc/rc.d/rc.local > /dev/null 2>&1 fi + if test -f /etc/rc.local.columnstoreSave ; then $sudo mv -f /etc/rc.local.columnstoreSave /etc/rc.local > /dev/null 2>&1 fi @@ -65,23 +75,23 @@ fi #remove log file directories #rm -rf /var/log/mariadb/columnstore > /dev/null 2>&1 #rm -f $installdir/mysql/db/*.err > /dev/null 2>&1 -rm -f /var/log/mariadb/columnstore/activeAlarms > /dev/null 2>&1 -rm -f /var/log/mariadb/columnstore/*.log1 > /dev/null 2>&1 +sudo rm -f /var/log/mariadb/columnstore/activeAlarms > /dev/null 2>&1 +sudo rm -f /var/log/mariadb/columnstore/*.log1 > /dev/null 2>&1 rm -rf $installdir/mysql/db/columnstore_log_archive > /dev/null 2>&1 sudo rm -f /etc/default/columnstore # remove Mariab Columnstore Log Rotate File abd Transaction Log File -rm -f /etc/logrotate.d/columnstore > /dev/null 2>&1 -rm -f /etc/cron.d/transactionLog > /dev/null 2>&1 +sudo rm -f /etc/logrotate.d/columnstore > /dev/null 2>&1 +sudo rm -f /etc/cron.d/transactionLog > /dev/null 2>&1 # delete Mariab Columnstore shared memory segments $installdir/bin/clearShm > /dev/null 2>&1 # delete prat files -rm -f /etc/cron.d/ps > /dev/null 2>&1 -rm -f /etc/pscollect > /dev/null 2>&1 -/etc/init.d/crond reload > /dev/null 2>&1 -sudo systemctl reload crond.service > /dev/null 2>&1 +sudo rm -f /etc/cron.d/ps > /dev/null 2>&1 +sudo rm -f /etc/pscollect > /dev/null 2>&1 +sudo /etc/init.d/crond reload > /dev/null 2>&1 +sudo sudo systemctl reload crond.service > /dev/null 2>&1 # delete tmp files rm -f $installdir/local/*.columnstore @@ -90,7 +100,7 @@ rm -rf /tmp/bucketreuse rm -f /tmp/columnstore.txt rm -f /tmp/dbbuilder.* rm -f /tmp/dbrmfiles -rm -f /var/lock/subsys/columnstore +sudo rm -f /var/lock/subsys/columnstore rm -f /tmp/pkgcheck rm -f /tmp/upgrade-status.log.* rm -f /tmp/mount.log @@ -122,8 +132,8 @@ systemctl=`which systemctl 2>/dev/null` if [ -n "$systemctl" ]; then sudo systemctl disable columnstore >/dev/null 2>&1 - rm -f /usr/lib/systemd/system/columnstore.service - rm -f /lib/systemd/system/columnstore.service + sudo rm -f /usr/lib/systemd/system/columnstore.service + sudo rm -f /lib/systemd/system/columnstore.service else chkconfig=`which chkconfig 2>/dev/null` if [ -n "$chkconfig" ]; then @@ -158,9 +168,9 @@ fi #remove library paths if [ $installdir == "/usr/local/mariadb/columnstore" ]; then # remove library config file - rm -f /etc/ld.so.conf.d/columnstore.conf - rm -f /etc/ld.so.cache - ldconfig + sudo rm -f /etc/ld.so.conf.d/columnstore.conf + sudo rm -f /etc/ld.so.cache + sudo ldconfig fi #tell user to run post configure script diff --git a/oam/oamcpp/liboamcpp.cpp b/oam/oamcpp/liboamcpp.cpp index efd2cc977..55dbdefea 100644 --- a/oam/oamcpp/liboamcpp.cpp +++ b/oam/oamcpp/liboamcpp.cpp @@ -9614,7 +9614,7 @@ namespace oam // not root user // The stat above may fail for non-root because of permissions // This is a non-optimal solution - string cmd = "pgrep ProcMon"; + string cmd = "pgrep ProcMon > /dev/null 2>&1"; if (system(cmd.c_str()) == 0) { return true; diff --git a/oamapps/alarmmanager/alarmmanager.cpp b/oamapps/alarmmanager/alarmmanager.cpp index 499cc7755..7d33b0302 100644 --- a/oamapps/alarmmanager/alarmmanager.cpp +++ b/oamapps/alarmmanager/alarmmanager.cpp @@ -234,7 +234,6 @@ void processAlarm(const Alarm& calAlarm) rewriteActiveLog (alarmList); } catch (runtime_error& e) { - if (ALARM_DEBUG) { LoggingID lid(11); MessageLog ml(lid); Message msg; @@ -242,9 +241,7 @@ void processAlarm(const Alarm& calAlarm) args.add("rewriteActiveLog error:"); args.add(e.what()); msg.format(args); - ml.logDebugMessage(msg); - } - exit(1); + ml.logErrorMessage(msg); } break; } @@ -256,7 +253,6 @@ void processAlarm(const Alarm& calAlarm) logAlarm (calAlarm, ACTIVE_ALARM_FILE); } catch (runtime_error& e) { - if (ALARM_DEBUG) { LoggingID lid(11); MessageLog ml(lid); Message msg; @@ -264,9 +260,7 @@ void processAlarm(const Alarm& calAlarm) args.add("logAlarm error:"); args.add(e.what()); msg.format(args); - ml.logDebugMessage(msg); - } - exit(1); + ml.logErrorMessage(msg); } } @@ -276,7 +270,6 @@ void processAlarm(const Alarm& calAlarm) logAlarm (calAlarm, ALARM_FILE); } catch (runtime_error& e) { - if (ALARM_DEBUG) { LoggingID lid(11); MessageLog ml(lid); Message msg; @@ -284,9 +277,7 @@ void processAlarm(const Alarm& calAlarm) args.add("logAlarm error:"); args.add(e.what()); msg.format(args); - ml.logDebugMessage(msg); - } - exit(1); + ml.logErrorMessage(msg); } } } @@ -462,7 +453,6 @@ void ALARMManager::sendAlarmReport (const char* componentID, int alarmID, int st configAlarm (calAlarm); } catch (runtime_error& e) { - if (ALARM_DEBUG) { LoggingID lid(11); MessageLog ml(lid); Message msg; @@ -470,9 +460,7 @@ void ALARMManager::sendAlarmReport (const char* componentID, int alarmID, int st args.add("configAlarm error:"); args.add(e.what()); msg.format(args); - ml.logDebugMessage(msg); - } - exit(1); + ml.logErrorMessage(msg); } return; diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index c8353d478..de2359000 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -814,10 +814,7 @@ int main(int argc, char *argv[]) //amazon install setup check bool amazonInstall = false; - if (rootUser) - system("ec2-version > /tmp/amazon.log 2>&1"); - else - system("sudo ec2-version > /tmp/amazon.log 2>&1"); + system("ec2-version > /tmp/amazon.log 2>&1"); ifstream in("/tmp/amazon.log"); @@ -3401,7 +3398,7 @@ int main(int argc, char *argv[]) //run remote command script cout << endl << "----- Starting MariaDB Columnstore on '" + remoteModuleName + "' -----" << endl << endl; - cmd = installDir + "/bin/remote_scp_put.sh " + remoteModuleIP + " " + installDir + "/etc/Columnstore.xml " + installDir + "/etc/. > /dev/null 2>&1"; + cmd = installDir + "/bin/remote_scp_put.sh " + remoteModuleIP + " " + installDir + "/etc/Columnstore.xml > /dev/null 2>&1"; int rtnCode = system(cmd.c_str()); cmd = installDir + "/bin/remote_command.sh " + remoteModuleIP + " " + password + @@ -5044,10 +5041,10 @@ bool updateBash() if (!rootUser) { - string cmd = "echo export columnstore_INSTALL_DIR=" + installDir + " >> " + fileName; + string cmd = "echo export COLUMNSTORE_INSTALL_DIR=" + installDir + " >> " + fileName; system(cmd.c_str()); - cmd = "echo export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$columnstore_INSTALL_DIR/lib:$columnstore_INSTALL_DIR/mysql/lib/mysql >> " + fileName; + cmd = "echo export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$COLUMNSTORE_INSTALL_DIR/lib:$COLUMNSTORE_INSTALL_DIR/mysql/lib/mysql >> " + fileName; system(cmd.c_str()); } From 75fc2d9ae020353d84555ec4af6592d27854a6d2 Mon Sep 17 00:00:00 2001 From: David Hill Date: Thu, 17 Nov 2016 14:42:53 +0000 Subject: [PATCH 8/8] mcol-404 - change the way that the log dir is setup --- oam/install_scripts/post-install | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/oam/install_scripts/post-install b/oam/install_scripts/post-install index 6e245b31b..620c7ec87 100755 --- a/oam/install_scripts/post-install +++ b/oam/install_scripts/post-install @@ -95,15 +95,9 @@ sudo ln -s /usr/lib64/perl5/CORE/libperl.so /usr/lib64/libperl.so >/dev/null 2>& cd / -if [ $user != "root" ]; then - sudo chmod 777 /var - sudo chmod 777 /var/log -fi - -test -d /var/log/mariadb || mkdir /var/log/mariadb >/dev/null 2>&1 -sudo chmod 777 /var/log/mariadb -test -d /var/log/mariadb/columnstore || mkdir /var/log/mariadb/columnstore >/dev/null 2>&1 -sudo chmod 777 /var/log/mariadb/columnstore +test -d /var/log/mariadb || sudo mkdir /var/log/mariadb >/dev/null 2>&1 +test -d /var/log/mariadb/columnstore || sudo mkdir /var/log/mariadb/columnstore >/dev/null 2>&1 +sudo chmod -R 777 /var/log/mariadb test -d /var/log/mariadb/columnstore/archive || mkdir /var/log/mariadb/columnstore/archive >/dev/null 2>&1 test -d /var/log/mariadb/columnstore/corefiles || mkdir /var/log/mariadb/columnstore/corefiles >/dev/null 2>&1