Fix arrow linkage on external project

debian/mariadb-plugin-columnstore.install

@@ -34,6 +34,8 @@ usr/bin/mariadb-columnstore-start.sh
 usr/bin/mariadb-columnstore-stop.sh
 usr/bin/mcs-savebrm.py
 usr/bin/mcs-loadbrm.py
+usr/bin/mcs_parquet_ddl
+usr/bin/mcs_parquet_gen
 usr/bin/mcs-stop-controllernode.sh
 usr/bin/mcsGetConfig
 usr/bin/mcsSetConfig

mysql-test/columnstore/basic/r/mcol-5505-parquet-ddl.result

@@ -15,9 +15,9 @@ parquet_test_table	CREATE TABLE `parquet_test_table` (
   `col9` varchar(2000) DEFAULT NULL,
   `col10` varchar(2000) DEFAULT NULL,
   `col11` varchar(2000) DEFAULT NULL,
-  `col12` timestamp(3) NULL DEFAULT NULL,
+  `col12` timestamp(3) NOT NULL DEFAULT current_timestamp(3) ON UPDATE current_timestamp(3),
   `col13` date DEFAULT NULL,
-  `col14` timestamp(3) NULL DEFAULT NULL,
+  `col14` timestamp(3) NOT NULL DEFAULT '0000-00-00 00:00:00.000',
   `col15` smallint(6) DEFAULT NULL,
   `col16` tinyint(4) DEFAULT NULL,
   `col17` decimal(9,3) DEFAULT NULL,
@@ -28,9 +28,9 @@ parquet_test_table	CREATE TABLE `parquet_test_table` (
   `col22` tinyint(1) DEFAULT NULL,
   `col23` decimal(38,10) DEFAULT NULL,
   `col24` time(6) DEFAULT NULL,
-  `col25` timestamp(6) NULL DEFAULT NULL,
+  `col25` timestamp(6) NOT NULL DEFAULT '0000-00-00 00:00:00.000000',
-  `col26` timestamp(6) NULL DEFAULT NULL,
+  `col26` timestamp(6) NOT NULL DEFAULT '0000-00-00 00:00:00.000000',
   `col27` varbinary(8000) DEFAULT NULL,
   `col28` char(4) DEFAULT NULL
-) ENGINE=Columnstore DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci
+) ENGINE=Columnstore DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_general_ci
 DROP DATABASE mcol_5505_parquet_ddl;

tools/parquetDDL/CMakeLists.txt

@@ -2,5 +2,5 @@ include_directories(${ENGINE_COMMON_INCLUDES})
 
 set(parquetDDL_SRCS main.cpp)
 add_executable(mcs_parquet_ddl ${parquetDDL_SRCS})
-target_link_libraries(mcs_parquet_ddl arrow parquet)
+target_link_libraries(mcs_parquet_ddl parquet)
 install(TARGETS mcs_parquet_ddl DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine)

tools/parquetGen/CMakeLists.txt

@@ -2,5 +2,5 @@ include_directories(${ENGINE_COMMON_INCLUDES})
 
 set(parquetGen_SRCS main.cpp)
 add_executable(mcs_parquet_gen ${parquetGen_SRCS})
-target_link_libraries(mcs_parquet_gen boost_system boost_filesystem arrow parquet)
+target_link_libraries(mcs_parquet_gen boost_system boost_filesystem parquet)
 install(TARGETS mcs_parquet_gen DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine)

tools/parquetGen/main.cpp

@@ -39,7 +39,7 @@ void generateIntTable(std::string fileDir)
   for (int32_t i = 0; i < reserve_num-1; i++)
     values.push_back(i);
 
-  values.push_back(2147483648);
+  values.push_back(static_cast<int32_t>(2147483648));
   PARQUET_THROW_NOT_OK(builder.AppendValues(values, validity));
   std::shared_ptr<arrow::Array> array;
   PARQUET_THROW_NOT_OK(builder.Finish(&array));
@@ -144,7 +144,7 @@ void generateDoubleTable(std::string fileDir)
   std::shared_ptr<arrow::io::FileOutputStream> outfile;
   PARQUET_ASSIGN_OR_THROW(
       outfile, arrow::io::FileOutputStream::Open(fileDir + "/double.parquet"));
-  PARQUET_THROW_NOT_OK(parquet::arrow::WriteTable(*table, pool, outfile, 3));	
+  PARQUET_THROW_NOT_OK(parquet::arrow::WriteTable(*table, pool, outfile, 3));
   PARQUET_THROW_NOT_OK(outfile->Close());
 }
 
@@ -173,7 +173,7 @@ void generateTimeTable(std::string fileDir)
   std::shared_ptr<arrow::io::FileOutputStream> outfile;
   PARQUET_ASSIGN_OR_THROW(
       outfile, arrow::io::FileOutputStream::Open(fileDir + "/time.parquet"));
-  PARQUET_THROW_NOT_OK(parquet::arrow::WriteTable(*table, pool, outfile, 3));	
+  PARQUET_THROW_NOT_OK(parquet::arrow::WriteTable(*table, pool, outfile, 3));
   PARQUET_THROW_NOT_OK(outfile->Close());
 }
 
@@ -202,7 +202,7 @@ void generateTime64Table(std::string fileDir)
   std::shared_ptr<arrow::io::FileOutputStream> outfile;
   PARQUET_ASSIGN_OR_THROW(
       outfile, arrow::io::FileOutputStream::Open(fileDir + "/time64.parquet"));
-  PARQUET_THROW_NOT_OK(parquet::arrow::WriteTable(*table, pool, outfile, 3));	
+  PARQUET_THROW_NOT_OK(parquet::arrow::WriteTable(*table, pool, outfile, 3));
   PARQUET_THROW_NOT_OK(outfile->Close());
 }
 
@@ -270,7 +270,7 @@ void generateStringTable(std::string fileDir)
   std::shared_ptr<arrow::io::FileOutputStream> outfile;
   PARQUET_ASSIGN_OR_THROW(
       outfile, arrow::io::FileOutputStream::Open(fileDir + "/string.parquet"));
-  PARQUET_THROW_NOT_OK(parquet::arrow::WriteTable(*table, pool, outfile, 3));	
+  PARQUET_THROW_NOT_OK(parquet::arrow::WriteTable(*table, pool, outfile, 3));
   PARQUET_THROW_NOT_OK(outfile->Close());
 }
 
@@ -299,7 +299,7 @@ void generateTimestampTable(std::string fileDir)
   std::shared_ptr<arrow::io::FileOutputStream> outfile;
   PARQUET_ASSIGN_OR_THROW(
       outfile, arrow::io::FileOutputStream::Open(fileDir + "/ts.parquet"));
-  PARQUET_THROW_NOT_OK(parquet::arrow::WriteTable(*table, pool, outfile, 3));	
+  PARQUET_THROW_NOT_OK(parquet::arrow::WriteTable(*table, pool, outfile, 3));
   PARQUET_THROW_NOT_OK(outfile->Close());
 }
 
@@ -328,7 +328,7 @@ void generateTimestampUsTable(std::string fileDir)
   std::shared_ptr<arrow::io::FileOutputStream> outfile;
   PARQUET_ASSIGN_OR_THROW(
       outfile, arrow::io::FileOutputStream::Open(fileDir + "/ts.parquet"));
-  PARQUET_THROW_NOT_OK(parquet::arrow::WriteTable(*table, pool, outfile, 3));	
+  PARQUET_THROW_NOT_OK(parquet::arrow::WriteTable(*table, pool, outfile, 3));
   PARQUET_THROW_NOT_OK(outfile->Close());
 }
 
@@ -357,7 +357,7 @@ void generateDateTable(std::string fileDir)
   std::shared_ptr<arrow::io::FileOutputStream> outfile;
   PARQUET_ASSIGN_OR_THROW(
       outfile, arrow::io::FileOutputStream::Open(fileDir + "/date.parquet"));
-  PARQUET_THROW_NOT_OK(parquet::arrow::WriteTable(*table, pool, outfile, 3));	
+  PARQUET_THROW_NOT_OK(parquet::arrow::WriteTable(*table, pool, outfile, 3));
   PARQUET_THROW_NOT_OK(outfile->Close());
 }
 
@@ -386,7 +386,7 @@ void generateInt16Table(std::string fileDir)
   std::shared_ptr<arrow::io::FileOutputStream> outfile;
   PARQUET_ASSIGN_OR_THROW(
       outfile, arrow::io::FileOutputStream::Open(fileDir + "/int16.parquet"));
-  PARQUET_THROW_NOT_OK(parquet::arrow::WriteTable(*table, pool, outfile, 3));	
+  PARQUET_THROW_NOT_OK(parquet::arrow::WriteTable(*table, pool, outfile, 3));
   PARQUET_THROW_NOT_OK(outfile->Close());
 }
 
@@ -416,7 +416,7 @@ void generateInt8Table(std::string fileDir)
   std::shared_ptr<arrow::io::FileOutputStream> outfile;
   PARQUET_ASSIGN_OR_THROW(
       outfile, arrow::io::FileOutputStream::Open(fileDir + "/int8.parquet"));
-  PARQUET_THROW_NOT_OK(parquet::arrow::WriteTable(*table, pool, outfile, 3));	
+  PARQUET_THROW_NOT_OK(parquet::arrow::WriteTable(*table, pool, outfile, 3));
   PARQUET_THROW_NOT_OK(outfile->Close());
 }
 
@@ -445,7 +445,7 @@ void generateDecimalTable(std::string fileDir)
   std::shared_ptr<arrow::io::FileOutputStream> outfile;
   PARQUET_ASSIGN_OR_THROW(
       outfile, arrow::io::FileOutputStream::Open(fileDir + "/decimal.parquet"));
-  PARQUET_THROW_NOT_OK(parquet::arrow::WriteTable(*table, pool, outfile, 3));	
+  PARQUET_THROW_NOT_OK(parquet::arrow::WriteTable(*table, pool, outfile, 3));
   PARQUET_THROW_NOT_OK(outfile->Close());
 }
 
@@ -508,7 +508,7 @@ void generateUint16Table(std::string fileDir)
   std::shared_ptr<arrow::io::FileOutputStream> outfile;
   PARQUET_ASSIGN_OR_THROW(
       outfile, arrow::io::FileOutputStream::Open(fileDir + "/uint16.parquet"));
-  PARQUET_THROW_NOT_OK(parquet::arrow::WriteTable(*table, pool, outfile, 3));	
+  PARQUET_THROW_NOT_OK(parquet::arrow::WriteTable(*table, pool, outfile, 3));
   PARQUET_THROW_NOT_OK(outfile->Close());
 }
 
@@ -537,7 +537,7 @@ void generateUint8Table(std::string fileDir)
   std::shared_ptr<arrow::io::FileOutputStream> outfile;
   PARQUET_ASSIGN_OR_THROW(
       outfile, arrow::io::FileOutputStream::Open(fileDir + "/uint8.parquet"));
-  PARQUET_THROW_NOT_OK(parquet::arrow::WriteTable(*table, pool, outfile, 3));	
+  PARQUET_THROW_NOT_OK(parquet::arrow::WriteTable(*table, pool, outfile, 3));
   PARQUET_THROW_NOT_OK(outfile->Close());
 }
 
@@ -670,7 +670,7 @@ void generateAllTable(std::string fileDir)
   PARQUET_THROW_NOT_OK(boolBuilder.AppendValues(boolValues, boolValidity));
   std::shared_ptr<arrow::Array> boolArray;
   PARQUET_THROW_NOT_OK(boolBuilder.Finish(&boolArray));
-  
+
   // int32
   arrow::Int32Builder int32Builder;
   PARQUET_THROW_NOT_OK(int32Builder.Reserve(reserve_num));
@@ -681,7 +681,7 @@ void generateAllTable(std::string fileDir)
   std::vector<int32_t> int32Values;
   for (int32_t i = 0; i < reserve_num-1; i++)
     int32Values.push_back(i);
-  int32Values.push_back(2147483648);
+  int32Values.push_back(static_cast<int32_t>(2147483648));
   PARQUET_THROW_NOT_OK(int32Builder.AppendValues(int32Values, int32Validity));
   std::shared_ptr<arrow::Array> int32Array;
   PARQUET_THROW_NOT_OK(int32Builder.Finish(&int32Array));
@@ -994,7 +994,7 @@ void generateAllTable(std::string fileDir)
   PARQUET_THROW_NOT_OK(binaryBuilder.AppendValues(binaryValues, binaryValidity));
   std::shared_ptr<arrow::Array> binaryArray;
   PARQUET_THROW_NOT_OK(binaryBuilder.Finish(&binaryArray));
-  
+
   // fixed_size_binary_array
   auto tfixedSizeType = arrow::FixedSizeBinaryType::Make(4);
   PARQUET_ASSIGN_OR_THROW(auto fixedSizeType, tfixedSizeType);

writeengine/bulk/CMakeLists.txt

@@ -30,7 +30,7 @@ add_library(we_bulk STATIC ${we_bulk_STAT_SRCS})
 
 add_dependencies(we_bulk loggingcpp)
 
-target_link_libraries(we_bulk ${NETSNMP_LIBRARIES})
+target_link_libraries(we_bulk ${NETSNMP_LIBRARIES} parquet)
 
 REMOVE_DEFINITIONS(-D_FILE_OFFSET_BITS=64)
 
@@ -42,9 +42,5 @@ add_executable(cpimport.bin ${cpimport.bin_SRCS})
 add_dependencies(cpimport.bin marias3)
 
 target_link_libraries(cpimport.bin ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS} ${S3API_DEPS} we_bulk we_xml)
-FIND_PACKAGE(Arrow)
-FIND_PACKAGE(Parquet)
-target_link_libraries(cpimport.bin arrow)
-target_link_libraries(cpimport.bin parquet)
 install(TARGETS cpimport.bin DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine)
 

writeengine/dictionary/we_dctnry.cpp

@@ -49,6 +49,9 @@ using namespace BRM;
 #include "IDBDataFile.h"
 #include "IDBPolicy.h"
 #include "cacheutils.h"
+
+#include <arrow/api.h>
+
 using namespace idbdatafile;
 #include "checks.h"
 #include "utils_utf8.h" // for utf8_truncate_point()

writeengine/dictionary/we_dctnry.h

@@ -37,10 +37,16 @@
 #include "bytestream.h"
 #include "nullstring.h"
 
-#include <arrow/api.h>
+
 
 #define EXPORT
 
+
+namespace arrow
+{
+  class Array;
+}
+
 /** Namespace WriteEngine */
 namespace WriteEngine
 {
@@ -162,7 +168,7 @@ class Dctnry : public DbFileOp
   /**
    * @brief Insert signature value to a file block and return token/pointer
    * (for Bulk use)
-   * 
+   *
    * @param columnData  - arrow array containing strings to be parsed
    * @param startRowIdx - start position for current batch parquet data
    * @param totalRow    - total number of rows in buf
@@ -172,7 +178,7 @@ class Dctnry : public DbFileOp
   EXPORT int insertDctnryParquet(std::shared_ptr<arrow::Array> columnData, int startRowIdx, const int totalRow,
                                  const int col, char* tokenBuf, long long& truncCount,
                                  const CHARSET_INFO* cs, const WriteEngine::ColType& weType);
-  
+
   /**
    * @brief Insert a signature value to a file block and return token/pointer
    * (for Bulk use)

writeengine/wrapper/CMakeLists.txt

@@ -41,6 +41,6 @@ add_library(writeengine SHARED ${writeengine_LIB_SRCS})
 
 add_dependencies(writeengine loggingcpp)
 
-target_link_libraries(writeengine ${NETSNMP_LIBRARIES})
+target_link_libraries(writeengine ${NETSNMP_LIBRARIES} arrow)
 
 install(TARGETS writeengine DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-engine)