diff --git a/debian/mariadb-plugin-columnstore.install b/debian/mariadb-plugin-columnstore.install index 1c609f6b2..b791374d8 100644 --- a/debian/mariadb-plugin-columnstore.install +++ b/debian/mariadb-plugin-columnstore.install @@ -34,6 +34,8 @@ usr/bin/mariadb-columnstore-start.sh usr/bin/mariadb-columnstore-stop.sh usr/bin/mcs-savebrm.py usr/bin/mcs-loadbrm.py +usr/bin/mcs_parquet_ddl +usr/bin/mcs_parquet_gen usr/bin/mcs-stop-controllernode.sh usr/bin/mcsGetConfig usr/bin/mcsSetConfig diff --git a/mysql-test/columnstore/basic/r/mcol-5505-parquet-ddl.result b/mysql-test/columnstore/basic/r/mcol-5505-parquet-ddl.result index 7695bf7b3..db627a7ae 100644 --- a/mysql-test/columnstore/basic/r/mcol-5505-parquet-ddl.result +++ b/mysql-test/columnstore/basic/r/mcol-5505-parquet-ddl.result @@ -15,9 +15,9 @@ parquet_test_table CREATE TABLE `parquet_test_table` ( `col9` varchar(2000) DEFAULT NULL, `col10` varchar(2000) DEFAULT NULL, `col11` varchar(2000) DEFAULT NULL, - `col12` timestamp(3) NULL DEFAULT NULL, + `col12` timestamp(3) NOT NULL DEFAULT current_timestamp(3) ON UPDATE current_timestamp(3), `col13` date DEFAULT NULL, - `col14` timestamp(3) NULL DEFAULT NULL, + `col14` timestamp(3) NOT NULL DEFAULT '0000-00-00 00:00:00.000', `col15` smallint(6) DEFAULT NULL, `col16` tinyint(4) DEFAULT NULL, `col17` decimal(9,3) DEFAULT NULL, @@ -28,9 +28,9 @@ parquet_test_table CREATE TABLE `parquet_test_table` ( `col22` tinyint(1) DEFAULT NULL, `col23` decimal(38,10) DEFAULT NULL, `col24` time(6) DEFAULT NULL, - `col25` timestamp(6) NULL DEFAULT NULL, - `col26` timestamp(6) NULL DEFAULT NULL, + `col25` timestamp(6) NOT NULL DEFAULT '0000-00-00 00:00:00.000000', + `col26` timestamp(6) NOT NULL DEFAULT '0000-00-00 00:00:00.000000', `col27` varbinary(8000) DEFAULT NULL, `col28` char(4) DEFAULT NULL -) ENGINE=Columnstore DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +) ENGINE=Columnstore DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_general_ci DROP DATABASE mcol_5505_parquet_ddl; diff --git a/tools/parquetDDL/CMakeLists.txt b/tools/parquetDDL/CMakeLists.txt index b5b2f5740..16711bbff 100644 --- a/tools/parquetDDL/CMakeLists.txt +++ b/tools/parquetDDL/CMakeLists.txt @@ -2,5 +2,5 @@ include_directories(${ENGINE_COMMON_INCLUDES}) set(parquetDDL_SRCS main.cpp) add_executable(mcs_parquet_ddl ${parquetDDL_SRCS}) -target_link_libraries(mcs_parquet_ddl arrow parquet) +target_link_libraries(mcs_parquet_ddl parquet) install(TARGETS mcs_parquet_ddl DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) diff --git a/tools/parquetGen/CMakeLists.txt b/tools/parquetGen/CMakeLists.txt index c8a3c2ba7..0a901e432 100644 --- a/tools/parquetGen/CMakeLists.txt +++ b/tools/parquetGen/CMakeLists.txt @@ -2,5 +2,5 @@ include_directories(${ENGINE_COMMON_INCLUDES}) set(parquetGen_SRCS main.cpp) add_executable(mcs_parquet_gen ${parquetGen_SRCS}) -target_link_libraries(mcs_parquet_gen boost_system boost_filesystem arrow parquet) +target_link_libraries(mcs_parquet_gen boost_system boost_filesystem parquet) install(TARGETS mcs_parquet_gen DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) diff --git a/tools/parquetGen/main.cpp b/tools/parquetGen/main.cpp index 1142d426d..090891197 100644 --- a/tools/parquetGen/main.cpp +++ b/tools/parquetGen/main.cpp @@ -39,7 +39,7 @@ void generateIntTable(std::string fileDir) for (int32_t i = 0; i < reserve_num-1; i++) values.push_back(i); - values.push_back(2147483648); + values.push_back(static_cast(2147483648)); PARQUET_THROW_NOT_OK(builder.AppendValues(values, validity)); std::shared_ptr array; PARQUET_THROW_NOT_OK(builder.Finish(&array)); @@ -144,7 +144,7 @@ void generateDoubleTable(std::string fileDir) std::shared_ptr outfile; PARQUET_ASSIGN_OR_THROW( outfile, arrow::io::FileOutputStream::Open(fileDir + "/double.parquet")); - PARQUET_THROW_NOT_OK(parquet::arrow::WriteTable(*table, pool, outfile, 3)); + PARQUET_THROW_NOT_OK(parquet::arrow::WriteTable(*table, pool, outfile, 3)); PARQUET_THROW_NOT_OK(outfile->Close()); } @@ -173,7 +173,7 @@ void generateTimeTable(std::string fileDir) std::shared_ptr outfile; PARQUET_ASSIGN_OR_THROW( outfile, arrow::io::FileOutputStream::Open(fileDir + "/time.parquet")); - PARQUET_THROW_NOT_OK(parquet::arrow::WriteTable(*table, pool, outfile, 3)); + PARQUET_THROW_NOT_OK(parquet::arrow::WriteTable(*table, pool, outfile, 3)); PARQUET_THROW_NOT_OK(outfile->Close()); } @@ -202,7 +202,7 @@ void generateTime64Table(std::string fileDir) std::shared_ptr outfile; PARQUET_ASSIGN_OR_THROW( outfile, arrow::io::FileOutputStream::Open(fileDir + "/time64.parquet")); - PARQUET_THROW_NOT_OK(parquet::arrow::WriteTable(*table, pool, outfile, 3)); + PARQUET_THROW_NOT_OK(parquet::arrow::WriteTable(*table, pool, outfile, 3)); PARQUET_THROW_NOT_OK(outfile->Close()); } @@ -270,7 +270,7 @@ void generateStringTable(std::string fileDir) std::shared_ptr outfile; PARQUET_ASSIGN_OR_THROW( outfile, arrow::io::FileOutputStream::Open(fileDir + "/string.parquet")); - PARQUET_THROW_NOT_OK(parquet::arrow::WriteTable(*table, pool, outfile, 3)); + PARQUET_THROW_NOT_OK(parquet::arrow::WriteTable(*table, pool, outfile, 3)); PARQUET_THROW_NOT_OK(outfile->Close()); } @@ -299,7 +299,7 @@ void generateTimestampTable(std::string fileDir) std::shared_ptr outfile; PARQUET_ASSIGN_OR_THROW( outfile, arrow::io::FileOutputStream::Open(fileDir + "/ts.parquet")); - PARQUET_THROW_NOT_OK(parquet::arrow::WriteTable(*table, pool, outfile, 3)); + PARQUET_THROW_NOT_OK(parquet::arrow::WriteTable(*table, pool, outfile, 3)); PARQUET_THROW_NOT_OK(outfile->Close()); } @@ -328,7 +328,7 @@ void generateTimestampUsTable(std::string fileDir) std::shared_ptr outfile; PARQUET_ASSIGN_OR_THROW( outfile, arrow::io::FileOutputStream::Open(fileDir + "/ts.parquet")); - PARQUET_THROW_NOT_OK(parquet::arrow::WriteTable(*table, pool, outfile, 3)); + PARQUET_THROW_NOT_OK(parquet::arrow::WriteTable(*table, pool, outfile, 3)); PARQUET_THROW_NOT_OK(outfile->Close()); } @@ -357,7 +357,7 @@ void generateDateTable(std::string fileDir) std::shared_ptr outfile; PARQUET_ASSIGN_OR_THROW( outfile, arrow::io::FileOutputStream::Open(fileDir + "/date.parquet")); - PARQUET_THROW_NOT_OK(parquet::arrow::WriteTable(*table, pool, outfile, 3)); + PARQUET_THROW_NOT_OK(parquet::arrow::WriteTable(*table, pool, outfile, 3)); PARQUET_THROW_NOT_OK(outfile->Close()); } @@ -386,7 +386,7 @@ void generateInt16Table(std::string fileDir) std::shared_ptr outfile; PARQUET_ASSIGN_OR_THROW( outfile, arrow::io::FileOutputStream::Open(fileDir + "/int16.parquet")); - PARQUET_THROW_NOT_OK(parquet::arrow::WriteTable(*table, pool, outfile, 3)); + PARQUET_THROW_NOT_OK(parquet::arrow::WriteTable(*table, pool, outfile, 3)); PARQUET_THROW_NOT_OK(outfile->Close()); } @@ -416,7 +416,7 @@ void generateInt8Table(std::string fileDir) std::shared_ptr outfile; PARQUET_ASSIGN_OR_THROW( outfile, arrow::io::FileOutputStream::Open(fileDir + "/int8.parquet")); - PARQUET_THROW_NOT_OK(parquet::arrow::WriteTable(*table, pool, outfile, 3)); + PARQUET_THROW_NOT_OK(parquet::arrow::WriteTable(*table, pool, outfile, 3)); PARQUET_THROW_NOT_OK(outfile->Close()); } @@ -445,7 +445,7 @@ void generateDecimalTable(std::string fileDir) std::shared_ptr outfile; PARQUET_ASSIGN_OR_THROW( outfile, arrow::io::FileOutputStream::Open(fileDir + "/decimal.parquet")); - PARQUET_THROW_NOT_OK(parquet::arrow::WriteTable(*table, pool, outfile, 3)); + PARQUET_THROW_NOT_OK(parquet::arrow::WriteTable(*table, pool, outfile, 3)); PARQUET_THROW_NOT_OK(outfile->Close()); } @@ -508,7 +508,7 @@ void generateUint16Table(std::string fileDir) std::shared_ptr outfile; PARQUET_ASSIGN_OR_THROW( outfile, arrow::io::FileOutputStream::Open(fileDir + "/uint16.parquet")); - PARQUET_THROW_NOT_OK(parquet::arrow::WriteTable(*table, pool, outfile, 3)); + PARQUET_THROW_NOT_OK(parquet::arrow::WriteTable(*table, pool, outfile, 3)); PARQUET_THROW_NOT_OK(outfile->Close()); } @@ -537,7 +537,7 @@ void generateUint8Table(std::string fileDir) std::shared_ptr outfile; PARQUET_ASSIGN_OR_THROW( outfile, arrow::io::FileOutputStream::Open(fileDir + "/uint8.parquet")); - PARQUET_THROW_NOT_OK(parquet::arrow::WriteTable(*table, pool, outfile, 3)); + PARQUET_THROW_NOT_OK(parquet::arrow::WriteTable(*table, pool, outfile, 3)); PARQUET_THROW_NOT_OK(outfile->Close()); } @@ -670,7 +670,7 @@ void generateAllTable(std::string fileDir) PARQUET_THROW_NOT_OK(boolBuilder.AppendValues(boolValues, boolValidity)); std::shared_ptr boolArray; PARQUET_THROW_NOT_OK(boolBuilder.Finish(&boolArray)); - + // int32 arrow::Int32Builder int32Builder; PARQUET_THROW_NOT_OK(int32Builder.Reserve(reserve_num)); @@ -681,7 +681,7 @@ void generateAllTable(std::string fileDir) std::vector int32Values; for (int32_t i = 0; i < reserve_num-1; i++) int32Values.push_back(i); - int32Values.push_back(2147483648); + int32Values.push_back(static_cast(2147483648)); PARQUET_THROW_NOT_OK(int32Builder.AppendValues(int32Values, int32Validity)); std::shared_ptr int32Array; PARQUET_THROW_NOT_OK(int32Builder.Finish(&int32Array)); @@ -994,7 +994,7 @@ void generateAllTable(std::string fileDir) PARQUET_THROW_NOT_OK(binaryBuilder.AppendValues(binaryValues, binaryValidity)); std::shared_ptr binaryArray; PARQUET_THROW_NOT_OK(binaryBuilder.Finish(&binaryArray)); - + // fixed_size_binary_array auto tfixedSizeType = arrow::FixedSizeBinaryType::Make(4); PARQUET_ASSIGN_OR_THROW(auto fixedSizeType, tfixedSizeType); diff --git a/writeengine/bulk/CMakeLists.txt b/writeengine/bulk/CMakeLists.txt index 99aab1e13..a44c4fb7f 100644 --- a/writeengine/bulk/CMakeLists.txt +++ b/writeengine/bulk/CMakeLists.txt @@ -30,7 +30,7 @@ add_library(we_bulk STATIC ${we_bulk_STAT_SRCS}) add_dependencies(we_bulk loggingcpp) -target_link_libraries(we_bulk ${NETSNMP_LIBRARIES}) +target_link_libraries(we_bulk ${NETSNMP_LIBRARIES} parquet) REMOVE_DEFINITIONS(-D_FILE_OFFSET_BITS=64) @@ -42,9 +42,5 @@ add_executable(cpimport.bin ${cpimport.bin_SRCS}) add_dependencies(cpimport.bin marias3) target_link_libraries(cpimport.bin ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS} ${S3API_DEPS} we_bulk we_xml) -FIND_PACKAGE(Arrow) -FIND_PACKAGE(Parquet) -target_link_libraries(cpimport.bin arrow) -target_link_libraries(cpimport.bin parquet) install(TARGETS cpimport.bin DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) diff --git a/writeengine/dictionary/we_dctnry.cpp b/writeengine/dictionary/we_dctnry.cpp index 52e543847..7b4650abc 100644 --- a/writeengine/dictionary/we_dctnry.cpp +++ b/writeengine/dictionary/we_dctnry.cpp @@ -49,6 +49,9 @@ using namespace BRM; #include "IDBDataFile.h" #include "IDBPolicy.h" #include "cacheutils.h" + +#include + using namespace idbdatafile; #include "checks.h" #include "utils_utf8.h" // for utf8_truncate_point() diff --git a/writeengine/dictionary/we_dctnry.h b/writeengine/dictionary/we_dctnry.h index a9eca380b..5898748ac 100644 --- a/writeengine/dictionary/we_dctnry.h +++ b/writeengine/dictionary/we_dctnry.h @@ -37,10 +37,16 @@ #include "bytestream.h" #include "nullstring.h" -#include + #define EXPORT + +namespace arrow +{ + class Array; +} + /** Namespace WriteEngine */ namespace WriteEngine { @@ -162,7 +168,7 @@ class Dctnry : public DbFileOp /** * @brief Insert signature value to a file block and return token/pointer * (for Bulk use) - * + * * @param columnData - arrow array containing strings to be parsed * @param startRowIdx - start position for current batch parquet data * @param totalRow - total number of rows in buf @@ -172,7 +178,7 @@ class Dctnry : public DbFileOp EXPORT int insertDctnryParquet(std::shared_ptr columnData, int startRowIdx, const int totalRow, const int col, char* tokenBuf, long long& truncCount, const CHARSET_INFO* cs, const WriteEngine::ColType& weType); - + /** * @brief Insert a signature value to a file block and return token/pointer * (for Bulk use) diff --git a/writeengine/wrapper/CMakeLists.txt b/writeengine/wrapper/CMakeLists.txt index 6051d83c5..7bb63c26d 100644 --- a/writeengine/wrapper/CMakeLists.txt +++ b/writeengine/wrapper/CMakeLists.txt @@ -41,6 +41,6 @@ add_library(writeengine SHARED ${writeengine_LIB_SRCS}) add_dependencies(writeengine loggingcpp) -target_link_libraries(writeengine ${NETSNMP_LIBRARIES}) +target_link_libraries(writeengine ${NETSNMP_LIBRARIES} arrow) install(TARGETS writeengine DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-engine)