From 4cca5fa7fa4a71a107f43a222c10a141aa55a498 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Tue, 20 Jun 2017 19:08:48 +0100 Subject: [PATCH] MCOL-769 Much higher performance bulk insert Use void* pointers instead of boost::any with lots of copies --- mariadb-columnstore-engine.config | 2 + mariadb-columnstore-engine.creator | 1 + mariadb-columnstore-engine.files | 1333 ++++++++++++++++++++++ mariadb-columnstore-engine.includes | 81 ++ writeengine/server/we_dmlcommandproc.cpp | 177 +-- writeengine/wrapper/we_colop.h | 4 +- writeengine/wrapper/writeengine.cpp | 830 ++++++++++++++ writeengine/wrapper/writeengine.h | 19 + 8 files changed, 2330 insertions(+), 117 deletions(-) create mode 100644 mariadb-columnstore-engine.config create mode 100644 mariadb-columnstore-engine.creator create mode 100644 mariadb-columnstore-engine.files create mode 100644 mariadb-columnstore-engine.includes diff --git a/mariadb-columnstore-engine.config b/mariadb-columnstore-engine.config new file mode 100644 index 000000000..e0284f425 --- /dev/null +++ b/mariadb-columnstore-engine.config @@ -0,0 +1,2 @@ +// Add predefined macros for your project here. For example: +// #define THE_ANSWER 42 diff --git a/mariadb-columnstore-engine.creator b/mariadb-columnstore-engine.creator new file mode 100644 index 000000000..e94cbbd30 --- /dev/null +++ b/mariadb-columnstore-engine.creator @@ -0,0 +1 @@ +[General] diff --git a/mariadb-columnstore-engine.files b/mariadb-columnstore-engine.files new file mode 100644 index 000000000..19d697054 --- /dev/null +++ b/mariadb-columnstore-engine.files @@ -0,0 +1,1333 @@ +dbcon/ddlpackage/altertable.cpp +dbcon/ddlpackage/columndef.cpp +dbcon/ddlpackage/createindex.cpp +dbcon/ddlpackage/createtable.cpp +dbcon/ddlpackage/ddl-gram-win.cpp +dbcon/ddlpackage/ddl-gram-win.h +dbcon/ddlpackage/ddl-scan-win.cpp +dbcon/ddlpackage/ddlpkg.cpp +dbcon/ddlpackage/ddlpkg.h +dbcon/ddlpackage/dropindex.cpp +dbcon/ddlpackage/droppartition.cpp +dbcon/ddlpackage/droptable.cpp +dbcon/ddlpackage/gramtest.cpp +dbcon/ddlpackage/markpartition.cpp +dbcon/ddlpackage/old-tdriver.cpp +dbcon/ddlpackage/restorepartition.cpp +dbcon/ddlpackage/serialize.cpp +dbcon/ddlpackage/sqlparser.cpp +dbcon/ddlpackage/sqlparser.h +dbcon/ddlpackage/sqlstatement.cpp +dbcon/ddlpackage/sqlstatementlist.cpp +dbcon/ddlpackage/tabledef.cpp +dbcon/ddlpackage/tdriver.cpp +dbcon/ddlpackageproc/altertableprocessor.cpp +dbcon/ddlpackageproc/altertableprocessor.h +dbcon/ddlpackageproc/createindexprocessor.cpp +dbcon/ddlpackageproc/createindexprocessor.h +dbcon/ddlpackageproc/createtableprocessor.cpp +dbcon/ddlpackageproc/createtableprocessor.h +dbcon/ddlpackageproc/ddlindexpopulator.cpp +dbcon/ddlpackageproc/ddlindexpopulator.h +dbcon/ddlpackageproc/ddlpackageprocessor.cpp +dbcon/ddlpackageproc/ddlpackageprocessor.h +dbcon/ddlpackageproc/ddlpackageprocessorfactory.cpp +dbcon/ddlpackageproc/ddlpackageprocessorfactory.h +dbcon/ddlpackageproc/dropindexprocessor.cpp +dbcon/ddlpackageproc/dropindexprocessor.h +dbcon/ddlpackageproc/droppartitionprocessor.cpp +dbcon/ddlpackageproc/droppartitionprocessor.h +dbcon/ddlpackageproc/droptableprocessor.cpp +dbcon/ddlpackageproc/droptableprocessor.h +dbcon/ddlpackageproc/markpartitionprocessor.cpp +dbcon/ddlpackageproc/markpartitionprocessor.h +dbcon/ddlpackageproc/mydriver.cpp +dbcon/ddlpackageproc/resource.h +dbcon/ddlpackageproc/restorepartitionprocessor.cpp +dbcon/ddlpackageproc/restorepartitionprocessor.h +dbcon/ddlpackageproc/tdriver.cpp +dbcon/dmlpackage/calpontdmlfactory.cpp +dbcon/dmlpackage/calpontdmlfactory.h +dbcon/dmlpackage/calpontdmlpackage.cpp +dbcon/dmlpackage/calpontdmlpackage.h +dbcon/dmlpackage/commanddmlpackage.cpp +dbcon/dmlpackage/commanddmlpackage.h +dbcon/dmlpackage/deletedmlpackage.cpp +dbcon/dmlpackage/deletedmlpackage.h +dbcon/dmlpackage/dml-gram-win.cpp +dbcon/dmlpackage/dml-gram-win.h +dbcon/dmlpackage/dml-scan-win.cpp +dbcon/dmlpackage/dmlcolumn.cpp +dbcon/dmlpackage/dmlcolumn.h +dbcon/dmlpackage/dmlobject.cpp +dbcon/dmlpackage/dmlobject.h +dbcon/dmlpackage/dmlpackage.h +dbcon/dmlpackage/dmlparser.cpp +dbcon/dmlpackage/dmlparser.h +dbcon/dmlpackage/dmlpkg.cpp +dbcon/dmlpackage/dmlpkg.h +dbcon/dmlpackage/dmltable.cpp +dbcon/dmlpackage/dmltable.h +dbcon/dmlpackage/gramtest.cpp +dbcon/dmlpackage/insertdmlpackage.cpp +dbcon/dmlpackage/insertdmlpackage.h +dbcon/dmlpackage/mysqldmlstatement.cpp +dbcon/dmlpackage/mysqldmlstatement.h +dbcon/dmlpackage/oracledmlstatement.cpp +dbcon/dmlpackage/oracledmlstatement.h +dbcon/dmlpackage/row.cpp +dbcon/dmlpackage/row.h +dbcon/dmlpackage/tdriver.cpp +dbcon/dmlpackage/updatedmlpackage.cpp +dbcon/dmlpackage/updatedmlpackage.h +dbcon/dmlpackage/vendordmlstatement.cpp +dbcon/dmlpackage/vendordmlstatement.h +dbcon/dmlpackageproc/autoincrementdata.cpp +dbcon/dmlpackageproc/autoincrementdata.h +dbcon/dmlpackageproc/commandpackageprocessor.cpp +dbcon/dmlpackageproc/commandpackageprocessor.h +dbcon/dmlpackageproc/deletepackageprocessor.cpp +dbcon/dmlpackageproc/deletepackageprocessor.h +dbcon/dmlpackageproc/dmlpackageprocessor.cpp +dbcon/dmlpackageproc/dmlpackageprocessor.h +dbcon/dmlpackageproc/dmlpackageprocessorfactory.cpp +dbcon/dmlpackageproc/dmlpackageprocessorfactory.h +dbcon/dmlpackageproc/insertpackageprocessor.cpp +dbcon/dmlpackageproc/insertpackageprocessor.h +dbcon/dmlpackageproc/resource.h +dbcon/dmlpackageproc/tablelockdata.cpp +dbcon/dmlpackageproc/tablelockdata.h +dbcon/dmlpackageproc/tdriver.cpp +dbcon/dmlpackageproc/updatepackageprocessor.cpp +dbcon/dmlpackageproc/updatepackageprocessor.h +dbcon/doc/calpontsystemcatalog.h +dbcon/execplan/aggregatecolumn.cpp +dbcon/execplan/aggregatecolumn.h +dbcon/execplan/arithmeticcolumn.cpp +dbcon/execplan/arithmeticcolumn.h +dbcon/execplan/arithmeticoperator.cpp +dbcon/execplan/arithmeticoperator.h +dbcon/execplan/blocksize.h +dbcon/execplan/btdriver.cpp +dbcon/execplan/calpontexecutionplan.cpp +dbcon/execplan/calpontexecutionplan.h +dbcon/execplan/calpontexecutionplanfactory.cpp +dbcon/execplan/calpontexecutionplanfactory.h +dbcon/execplan/calpontselectexecutionplan.cpp +dbcon/execplan/calpontselectexecutionplan.h +dbcon/execplan/calpontsystemcatalog.cpp +dbcon/execplan/calpontsystemcatalog.h +dbcon/execplan/clientrotator.cpp +dbcon/execplan/clientrotator.h +dbcon/execplan/columnresult.h +dbcon/execplan/constantcolumn.cpp +dbcon/execplan/constantcolumn.h +dbcon/execplan/constantfilter.cpp +dbcon/execplan/constantfilter.h +dbcon/execplan/existsfilter.cpp +dbcon/execplan/existsfilter.h +dbcon/execplan/exp_templates.h +dbcon/execplan/expressionparser.cpp +dbcon/execplan/expressionparser.h +dbcon/execplan/filter.cpp +dbcon/execplan/filter.h +dbcon/execplan/functioncolumn.cpp +dbcon/execplan/functioncolumn.h +dbcon/execplan/groupconcatcolumn.cpp +dbcon/execplan/groupconcatcolumn.h +dbcon/execplan/intervalcolumn.cpp +dbcon/execplan/intervalcolumn.h +dbcon/execplan/logicoperator.cpp +dbcon/execplan/logicoperator.h +dbcon/execplan/mysqlexecutionplan.cpp +dbcon/execplan/mysqlexecutionplan.h +dbcon/execplan/njlcolumnresult.h +dbcon/execplan/objectidmanager.cpp +dbcon/execplan/objectidmanager.h +dbcon/execplan/objectreader.cpp +dbcon/execplan/objectreader.h +dbcon/execplan/operator.cpp +dbcon/execplan/operator.h +dbcon/execplan/oracleexecutionplan.cpp +dbcon/execplan/oracleexecutionplan.h +dbcon/execplan/outerjoinonfilter.cpp +dbcon/execplan/outerjoinonfilter.h +dbcon/execplan/parsetree.h +dbcon/execplan/predicateoperator.cpp +dbcon/execplan/predicateoperator.h +dbcon/execplan/pseudocolumn.cpp +dbcon/execplan/pseudocolumn.h +dbcon/execplan/range.cpp +dbcon/execplan/range.h +dbcon/execplan/returnedcolumn.cpp +dbcon/execplan/returnedcolumn.h +dbcon/execplan/rowcolumn.cpp +dbcon/execplan/rowcolumn.h +dbcon/execplan/selectfilter.cpp +dbcon/execplan/selectfilter.h +dbcon/execplan/sessionmanager.cpp +dbcon/execplan/sessionmanager.h +dbcon/execplan/sessionmonitor.cpp +dbcon/execplan/sessionmonitor.h +dbcon/execplan/simplecolumn.cpp +dbcon/execplan/simplecolumn.h +dbcon/execplan/simplecolumn_decimal.h +dbcon/execplan/simplecolumn_int.h +dbcon/execplan/simplecolumn_uint.h +dbcon/execplan/simplefilter.cpp +dbcon/execplan/simplefilter.h +dbcon/execplan/simplescalarfilter.cpp +dbcon/execplan/simplescalarfilter.h +dbcon/execplan/tdriver-oid.cpp +dbcon/execplan/tdriver-sm.cpp +dbcon/execplan/tdriver.cpp +dbcon/execplan/tdriver1.cpp +dbcon/execplan/tdriver10.cpp +dbcon/execplan/tdriver11.cpp +dbcon/execplan/tdriver12.cpp +dbcon/execplan/tdriver13.cpp +dbcon/execplan/tdriver14.cpp +dbcon/execplan/tdriver15.cpp +dbcon/execplan/tdriver16.cpp +dbcon/execplan/tdriver17.cpp +dbcon/execplan/tdriver18.cpp +dbcon/execplan/tdriver19.cpp +dbcon/execplan/tdriver2.cpp +dbcon/execplan/tdriver20.cpp +dbcon/execplan/tdriver21.cpp +dbcon/execplan/tdriver22.cpp +dbcon/execplan/tdriver23.cpp +dbcon/execplan/tdriver3.cpp +dbcon/execplan/tdriver4.cpp +dbcon/execplan/tdriver5.cpp +dbcon/execplan/tdriver6.cpp +dbcon/execplan/tdriver7.cpp +dbcon/execplan/tdriver8.cpp +dbcon/execplan/tdriver9.cpp +dbcon/execplan/treenode.cpp +dbcon/execplan/treenode.h +dbcon/execplan/treenodeimpl.cpp +dbcon/execplan/treenodeimpl.h +dbcon/execplan/vendorexecutionplan.cpp +dbcon/execplan/vendorexecutionplan.h +dbcon/execplan/windowfunctioncolumn.cpp +dbcon/execplan/windowfunctioncolumn.h +dbcon/joblist/anydatalist.cpp +dbcon/joblist/bandeddl.h +dbcon/joblist/batchprimitiveprocessor-jl.cpp +dbcon/joblist/batchprimitiveprocessor-jl.h +dbcon/joblist/bpp-jl.h +dbcon/joblist/bucketdl.h +dbcon/joblist/columncommand-jl.cpp +dbcon/joblist/columncommand-jl.h +dbcon/joblist/command-jl.cpp +dbcon/joblist/command-jl.h +dbcon/joblist/crossenginestep.cpp +dbcon/joblist/crossenginestep.h +dbcon/joblist/datalist.h +dbcon/joblist/datalistimpl.h +dbcon/joblist/dictstep-jl.cpp +dbcon/joblist/dictstep-jl.h +dbcon/joblist/diskjoinstep.cpp +dbcon/joblist/diskjoinstep.h +dbcon/joblist/distributedenginecomm.cpp +dbcon/joblist/distributedenginecomm.h +dbcon/joblist/elementcompression.h +dbcon/joblist/elementtype.cpp +dbcon/joblist/elementtype.h +dbcon/joblist/errorinfo.h +dbcon/joblist/expressionstep.cpp +dbcon/joblist/expressionstep.h +dbcon/joblist/fifo.h +dbcon/joblist/filtercommand-jl.cpp +dbcon/joblist/filtercommand-jl.h +dbcon/joblist/filterstep.cpp +dbcon/joblist/groupconcat.cpp +dbcon/joblist/groupconcat.h +dbcon/joblist/jl_logger.cpp +dbcon/joblist/jl_logger.h +dbcon/joblist/jlf_common.cpp +dbcon/joblist/jlf_common.h +dbcon/joblist/jlf_execplantojoblist.cpp +dbcon/joblist/jlf_execplantojoblist.h +dbcon/joblist/jlf_graphics.cpp +dbcon/joblist/jlf_graphics.h +dbcon/joblist/jlf_subquery.cpp +dbcon/joblist/jlf_subquery.h +dbcon/joblist/jlf_tuplejoblist.cpp +dbcon/joblist/jlf_tuplejoblist.h +dbcon/joblist/joblist.cpp +dbcon/joblist/joblist.h +dbcon/joblist/joblistfactory.cpp +dbcon/joblist/joblistfactory.h +dbcon/joblist/joblisttypes.h +dbcon/joblist/jobstep.cpp +dbcon/joblist/jobstep.h +dbcon/joblist/jobstepassociation.cpp +dbcon/joblist/largedatalist.h +dbcon/joblist/largehashjoin.cpp +dbcon/joblist/largehashjoin.h +dbcon/joblist/lbidlist.cpp +dbcon/joblist/lbidlist.h +dbcon/joblist/limitedorderby.cpp +dbcon/joblist/limitedorderby.h +dbcon/joblist/passthrucommand-jl.cpp +dbcon/joblist/passthrucommand-jl.h +dbcon/joblist/passthrustep.cpp +dbcon/joblist/pcolscan.cpp +dbcon/joblist/pcolstep.cpp +dbcon/joblist/pdictionary.cpp +dbcon/joblist/pdictionaryscan.cpp +dbcon/joblist/primitivemsg.cpp +dbcon/joblist/primitivemsg.h +dbcon/joblist/primitivestep.h +dbcon/joblist/pseudocc-jl.cpp +dbcon/joblist/pseudocc-jl.h +dbcon/joblist/resource.h +dbcon/joblist/resourcedistributor.cpp +dbcon/joblist/resourcedistributor.h +dbcon/joblist/resourcemanager.cpp +dbcon/joblist/resourcemanager.h +dbcon/joblist/rowestimator.cpp +dbcon/joblist/rowestimator.h +dbcon/joblist/rtscommand-jl.cpp +dbcon/joblist/rtscommand-jl.h +dbcon/joblist/subquerystep.cpp +dbcon/joblist/subquerystep.h +dbcon/joblist/subquerytransformer.cpp +dbcon/joblist/subquerytransformer.h +dbcon/joblist/tablecolumn.cpp +dbcon/joblist/tablecolumn.h +dbcon/joblist/tdriver-agg.cpp +dbcon/joblist/tdriver-bdl.cpp +dbcon/joblist/tdriver-bru.cpp +dbcon/joblist/tdriver-datalist.cpp +dbcon/joblist/tdriver-dec.cpp +dbcon/joblist/tdriver-deliver.cpp +dbcon/joblist/tdriver-filter.cpp +dbcon/joblist/tdriver-function.cpp +dbcon/joblist/tdriver-hashjoin.cpp +dbcon/joblist/tdriver-index.cpp +dbcon/joblist/tdriver-jobstep.cpp +dbcon/joblist/tdriver-pdict.cpp +dbcon/joblist/tdriver-zdl.cpp +dbcon/joblist/tdriver1.cpp +dbcon/joblist/tdriver2.cpp +dbcon/joblist/threadsafequeue.h +dbcon/joblist/timeset.h +dbcon/joblist/timestamp.cpp +dbcon/joblist/timestamp.h +dbcon/joblist/tuple-bps.cpp +dbcon/joblist/tupleaggregatestep.cpp +dbcon/joblist/tupleaggregatestep.h +dbcon/joblist/tupleannexstep.cpp +dbcon/joblist/tupleannexstep.h +dbcon/joblist/tupleconstantstep.cpp +dbcon/joblist/tupleconstantstep.h +dbcon/joblist/tuplehashjoin.cpp +dbcon/joblist/tuplehashjoin.h +dbcon/joblist/tuplehavingstep.cpp +dbcon/joblist/tuplehavingstep.h +dbcon/joblist/tupleunion.cpp +dbcon/joblist/tupleunion.h +dbcon/joblist/unique32generator.cpp +dbcon/joblist/unique32generator.h +dbcon/joblist/virtualtable.cpp +dbcon/joblist/virtualtable.h +dbcon/joblist/windowfunctionstep.cpp +dbcon/joblist/windowfunctionstep.h +dbcon/mysql/cmake_dummy.cc +dbcon/mysql/ha_autoi.cpp +dbcon/mysql/ha_calpont.cpp +dbcon/mysql/ha_calpont.h +dbcon/mysql/ha_calpont_ddl.cpp +dbcon/mysql/ha_calpont_dml.cpp +dbcon/mysql/ha_calpont_execplan.cpp +dbcon/mysql/ha_calpont_impl.cpp +dbcon/mysql/ha_calpont_impl.h +dbcon/mysql/ha_calpont_impl_if.h +dbcon/mysql/ha_calpont_partition.cpp +dbcon/mysql/ha_calpont_udf.cpp +dbcon/mysql/ha_exists_sub.cpp +dbcon/mysql/ha_from_sub.cpp +dbcon/mysql/ha_in_sub.cpp +dbcon/mysql/ha_pseudocolumn.cpp +dbcon/mysql/ha_scalar_sub.cpp +dbcon/mysql/ha_select_sub.cpp +dbcon/mysql/ha_subquery.h +dbcon/mysql/ha_view.cpp +dbcon/mysql/ha_view.h +dbcon/mysql/ha_window_function.cpp +dbcon/mysql/idb_mysql.h +dbcon/mysql/is_columnstore_columns.cpp +dbcon/mysql/is_columnstore_extents.cpp +dbcon/mysql/is_columnstore_files.cpp +dbcon/mysql/is_columnstore_tables.cpp +dbcon/mysql/resource.h +dbcon/mysql/sm.cpp +dbcon/mysql/sm.h +dbcon/mysql/tdriver1.cpp +dbcon/mysql/versionnumber.h +ddlproc/ddlproc.cpp +ddlproc/ddlproc.h +ddlproc/ddlprocessor.cpp +ddlproc/ddlprocessor.h +ddlproc/resource.h +ddlproc/tdriver.cpp +decomsvr/cli.cpp +decomsvr/quicklz.c +decomsvr/quicklz.h +decomsvr/resource.h +decomsvr/server.cpp +dmlproc/batchinsertprocessor.cpp +dmlproc/batchinsertprocessor.h +dmlproc/dmlproc.cpp +dmlproc/dmlproc.h +dmlproc/dmlprocessor.cpp +dmlproc/dmlprocessor.h +dmlproc/dmlresultbuffer.cpp +dmlproc/dmlresultbuffer.h +dmlproc/resource.h +exemgr/activestatementcounter.cpp +exemgr/activestatementcounter.h +exemgr/femsghandler.cpp +exemgr/femsghandler.h +exemgr/main.cpp +exemgr/resource.h +oam/oamcpp/liboamcpp.cpp +oam/oamcpp/liboamcpp.h +oam/oamcpp/oamcache.cpp +oam/oamcpp/oamcache.h +oam/oamcpp/resource.h +oam/oamcpp/tdriver.cpp +oam/replaytxnlog/replaytxnlog.cpp +oam/replaytxnlog/replaytxnlog.h +oamapps/alarmmanager/alarm.cpp +oamapps/alarmmanager/alarm.h +oamapps/alarmmanager/alarmglobal.h +oamapps/alarmmanager/alarmmanager.cpp +oamapps/alarmmanager/alarmmanager.h +oamapps/alarmmanager/tdriver.cpp +oamapps/calpontConsole/calpontConsole.cpp +oamapps/calpontConsole/calpontConsole.h +oamapps/calpontConsole/tdriver.cpp +oamapps/calpontDB/calpontDB.cpp +oamapps/calpontSupport/calpontSupport.cpp +oamapps/columnstoreDB/columnstoreDB.cpp +oamapps/columnstoreSupport/calpontSupport.cpp +oamapps/columnstoreSupport/columnstoreSupport.cpp +oamapps/hardwareMonitor/hardwareMonitor.cpp +oamapps/hardwareMonitor/hardwareMonitor.h +oamapps/mcsadmin/mcsadmin.cpp +oamapps/mcsadmin/mcsadmin.h +oamapps/postConfigure/amazonInstaller.cpp +oamapps/postConfigure/getMySQLpw.cpp +oamapps/postConfigure/helpers.cpp +oamapps/postConfigure/helpers.h +oamapps/postConfigure/installer.cpp +oamapps/postConfigure/mycnfUpgrade.cpp +oamapps/postConfigure/patchInstaller.cpp +oamapps/postConfigure/postConfigure.cpp +oamapps/replayTransactionLog/replaytransactionlog.cpp +oamapps/replayTransactionLog/tdriver.cpp +oamapps/resourceMonitor/hardwareMonitor.cpp +oamapps/resourceMonitor/resourceMonitor.cpp +oamapps/resourceMonitor/tdriver.cpp +oamapps/serverMonitor/UMAutoSync.cpp +oamapps/serverMonitor/cpuMonitor.cpp +oamapps/serverMonitor/dbhealthMonitor.cpp +oamapps/serverMonitor/diskMonitor.cpp +oamapps/serverMonitor/hardwareMonitor.cpp +oamapps/serverMonitor/main.cpp +oamapps/serverMonitor/memoryMonitor.cpp +oamapps/serverMonitor/msgProcessor.cpp +oamapps/serverMonitor/procmonMonitor.cpp +oamapps/serverMonitor/serverMonitor.cpp +oamapps/serverMonitor/serverMonitor.h +oamapps/sessionWalker/sessionwalker.cpp +oamapps/sessionWalker/tdriver.cpp +primitives/blockcache/bcTest.cpp +primitives/blockcache/blockcacheclient.cpp +primitives/blockcache/blockcacheclient.h +primitives/blockcache/blockrequestprocessor.cpp +primitives/blockcache/blockrequestprocessor.h +primitives/blockcache/fileblockrequestqueue.cpp +primitives/blockcache/fileblockrequestqueue.h +primitives/blockcache/filebuffer.cpp +primitives/blockcache/filebuffer.h +primitives/blockcache/filebuffermgr.cpp +primitives/blockcache/filebuffermgr.h +primitives/blockcache/filerequest.cpp +primitives/blockcache/filerequest.h +primitives/blockcache/fsutils.cpp +primitives/blockcache/fsutils.h +primitives/blockcache/iomanager.cpp +primitives/blockcache/iomanager.h +primitives/blockcache/stats.cpp +primitives/blockcache/stats.h +primitives/blockcache/tdriver.cpp +primitives/linux-port/column.cpp +primitives/linux-port/dictionary.cpp +primitives/linux-port/index.cpp +primitives/linux-port/primitiveprocessor.cpp +primitives/linux-port/primitiveprocessor.h +primitives/linux-port/print_dictblock.cpp +primitives/linux-port/print_indexlist.cpp +primitives/linux-port/print_indextree_subblock.cpp +primitives/linux-port/tdriver.cpp +primitives/primproc/batchprimitiveprocessor.cpp +primitives/primproc/batchprimitiveprocessor.h +primitives/primproc/bpp.h +primitives/primproc/bppseeder.cpp +primitives/primproc/bppseeder.h +primitives/primproc/bppsendthread.cpp +primitives/primproc/bppsendthread.h +primitives/primproc/columncommand.cpp +primitives/primproc/columncommand.h +primitives/primproc/command.cpp +primitives/primproc/command.h +primitives/primproc/dictstep.cpp +primitives/primproc/dictstep.h +primitives/primproc/filtercommand.cpp +primitives/primproc/filtercommand.h +primitives/primproc/logger.cpp +primitives/primproc/passthrucommand.cpp +primitives/primproc/passthrucommand.h +primitives/primproc/pp_logger.h +primitives/primproc/primitiveserver.cpp +primitives/primproc/primitiveserver.h +primitives/primproc/primproc.cpp +primitives/primproc/primproc.h +primitives/primproc/pseudocc.cpp +primitives/primproc/pseudocc.h +primitives/primproc/resource.h +primitives/primproc/rtscommand.cpp +primitives/primproc/rtscommand.h +primitives/primproc/tdriver-umsocksel.cpp +primitives/primproc/tdriver.cpp +primitives/primproc/udf.cpp +primitives/primproc/umsocketselector.cpp +primitives/primproc/umsocketselector.h +procmgr/main.cpp +procmgr/processmanager.cpp +procmgr/processmanager.h +procmon/main.cpp +procmon/processmonitor.cpp +procmon/processmonitor.h +tools/bincvt/li2bin.cpp +tools/brmtest/brmtest.cpp +tools/brmtest/locks.cpp +tools/cfread/cfread.cpp +tools/cfread/mtread.cpp +tools/clearShm/main.cpp +tools/cleartablelock/cleartablelock.cpp +tools/cleartablelock/cleartablelockthread.cpp +tools/cleartablelock/cleartablelockthread.h +tools/configMgt/autoConfigure.cpp +tools/configMgt/autoInstaller.cpp +tools/configMgt/configure.cpp +tools/configMgt/stackReleaseChecker.cpp +tools/configMgt/svnQuery.cpp +tools/cplogger/main.cpp +tools/dbbuilder/dbbuilder.cpp +tools/dbbuilder/dbbuilder.h +tools/dbbuilder/systemcatalog.cpp +tools/dbbuilder/systemcatalog.h +tools/dbbuilder/tpchpopulate.cpp +tools/dbbuilder/tpchpopulate.h +tools/dbbuilder/tpchschema.cpp +tools/dbbuilder/tpchschema.h +tools/dbloadxml/colxml.cpp +tools/dbloadxml/inputmgr.cpp +tools/dbloadxml/inputmgr.h +tools/dbloadxml/resource.h +tools/ddlcleanup/ddlcleanup.cpp +tools/ddldriver/ddldriver.cpp +tools/dmldriver/dmldriver.cpp +tools/dmldriver/dmlif.cpp +tools/dmldriver/dmlif.h +tools/dmldriver/tpchrf2.cpp +tools/dmldriver/tpchrf2.h +tools/editem/editem.cpp +tools/evalidx/evalidx.cpp +tools/fixdate/fixdate.cpp +tools/getConfig/main.cpp +tools/hdfsCheck/hdfsCheck.cpp +tools/idb_comp/main.cpp +tools/idbmeminfo/idbmeminfo.cpp +tools/notificationTester/client.cpp +tools/notificationTester/main.cpp +tools/pingproc/pingproc.cpp +tools/qfe/bison-win.cpp +tools/qfe/bison-win.h +tools/qfe/cli.cpp +tools/qfe/cseputils.cpp +tools/qfe/cseputils.h +tools/qfe/ddlstmts.cpp +tools/qfe/ddlstmts.h +tools/qfe/lex-win.cpp +tools/qfe/parsequery.cpp +tools/qfe/parsequery.h +tools/qfe/returnedrows.cpp +tools/qfe/returnedrows.h +tools/qfe/sendcsep.cpp +tools/qfe/sendcsep.h +tools/qfe/server.cpp +tools/qfe/socketio.cpp +tools/qfe/socketio.h +tools/qfe/socktype.h +tools/rebuildEM/main.cpp +tools/sendPlan/sendplan.cpp +tools/setConfig/main.cpp +tools/valprint/valprint.cpp +tools/vbgen/myrand.cpp +tools/vbgen/myrand.h +tools/vbgen/vbgen.cpp +tools/viewtablelock/viewtablelock.cpp +utils/batchloader/batchloader.cpp +utils/batchloader/batchloader.h +utils/cacheutils/cacheutils.cpp +utils/cacheutils/cacheutils.h +utils/common/MonitorProcMem.cpp +utils/common/MonitorProcMem.h +utils/common/atomicops.h +utils/common/branchpred.h +utils/common/cgroupconfigurator.cpp +utils/common/cgroupconfigurator.h +utils/common/fixedallocator.cpp +utils/common/fixedallocator.h +utils/common/hasher.h +utils/common/nullvaluemanip.cpp +utils/common/nullvaluemanip.h +utils/common/poolallocator.cpp +utils/common/poolallocator.h +utils/common/simpleallocator.h +utils/common/stlpoolallocator.h +utils/common/syncstream.h +utils/compress/idbcompress.cpp +utils/compress/idbcompress.h +utils/compress/version1.cpp +utils/compress/version1.h +utils/configcpp/config.h +utils/configcpp/configcpp.cpp +utils/configcpp/configcpp.h +utils/configcpp/configstream.cpp +utils/configcpp/configstream.h +utils/configcpp/md5/md32_common.h +utils/configcpp/md5/md5.h +utils/configcpp/md5/md5_dgst.c +utils/configcpp/md5/md5_locl.h +utils/configcpp/md5/md5_one.c +utils/configcpp/md5/mem_clr.c +utils/configcpp/resource.h +utils/configcpp/tdriver.cpp +utils/configcpp/tdriver2.cpp +utils/configcpp/versionnumber.h +utils/configcpp/writeonce.cpp +utils/configcpp/writeonce.h +utils/configcpp/xmlparser.cpp +utils/configcpp/xmlparser.h +utils/dataconvert/dataconvert.cpp +utils/dataconvert/dataconvert.h +utils/dataconvert/tdriver.cpp +utils/ddlcleanup/ddlcleanuputil.cpp +utils/ddlcleanup/ddlcleanuputil.h +utils/funcexp/func_abs.cpp +utils/funcexp/func_add_time.cpp +utils/funcexp/func_ascii.cpp +utils/funcexp/func_between.cpp +utils/funcexp/func_bitand.cpp +utils/funcexp/func_bitwise.cpp +utils/funcexp/func_case.cpp +utils/funcexp/func_cast.cpp +utils/funcexp/func_ceil.cpp +utils/funcexp/func_char.cpp +utils/funcexp/func_char_length.cpp +utils/funcexp/func_coalesce.cpp +utils/funcexp/func_concat.cpp +utils/funcexp/func_concat_ws.cpp +utils/funcexp/func_conv.cpp +utils/funcexp/func_crc32.cpp +utils/funcexp/func_date.cpp +utils/funcexp/func_date_add.cpp +utils/funcexp/func_date_format.cpp +utils/funcexp/func_day.cpp +utils/funcexp/func_dayname.cpp +utils/funcexp/func_dayofweek.cpp +utils/funcexp/func_dayofyear.cpp +utils/funcexp/func_div.cpp +utils/funcexp/func_elt.cpp +utils/funcexp/func_exp.cpp +utils/funcexp/func_extract.cpp +utils/funcexp/func_find_in_set.cpp +utils/funcexp/func_floor.cpp +utils/funcexp/func_from_days.cpp +utils/funcexp/func_from_unixtime.cpp +utils/funcexp/func_get_format.cpp +utils/funcexp/func_greatest.cpp +utils/funcexp/func_hex.cpp +utils/funcexp/func_hour.cpp +utils/funcexp/func_idbpartition.cpp +utils/funcexp/func_if.cpp +utils/funcexp/func_ifnull.cpp +utils/funcexp/func_in.cpp +utils/funcexp/func_inet_aton.cpp +utils/funcexp/func_inet_ntoa.cpp +utils/funcexp/func_insert.cpp +utils/funcexp/func_instr.cpp +utils/funcexp/func_isnull.cpp +utils/funcexp/func_last_day.cpp +utils/funcexp/func_lcase.cpp +utils/funcexp/func_least.cpp +utils/funcexp/func_left.cpp +utils/funcexp/func_length.cpp +utils/funcexp/func_lpad.cpp +utils/funcexp/func_ltrim.cpp +utils/funcexp/func_makedate.cpp +utils/funcexp/func_maketime.cpp +utils/funcexp/func_math.cpp +utils/funcexp/func_md5.cpp +utils/funcexp/func_microsecond.cpp +utils/funcexp/func_minute.cpp +utils/funcexp/func_mod.cpp +utils/funcexp/func_month.cpp +utils/funcexp/func_monthname.cpp +utils/funcexp/func_nullif.cpp +utils/funcexp/func_period_add.cpp +utils/funcexp/func_period_diff.cpp +utils/funcexp/func_pow.cpp +utils/funcexp/func_quarter.cpp +utils/funcexp/func_rand.cpp +utils/funcexp/func_regexp.cpp +utils/funcexp/func_repeat.cpp +utils/funcexp/func_replace.cpp +utils/funcexp/func_reverse.cpp +utils/funcexp/func_right.cpp +utils/funcexp/func_round.cpp +utils/funcexp/func_rpad.cpp +utils/funcexp/func_rtrim.cpp +utils/funcexp/func_sec_to_time.cpp +utils/funcexp/func_second.cpp +utils/funcexp/func_sha.cpp +utils/funcexp/func_sign.cpp +utils/funcexp/func_str_to_date.cpp +utils/funcexp/func_strcmp.cpp +utils/funcexp/func_substr.cpp +utils/funcexp/func_substring_index.cpp +utils/funcexp/func_sysdate.cpp +utils/funcexp/func_time.cpp +utils/funcexp/func_time_format.cpp +utils/funcexp/func_time_to_sec.cpp +utils/funcexp/func_timediff.cpp +utils/funcexp/func_timestampdiff.cpp +utils/funcexp/func_to_days.cpp +utils/funcexp/func_trim.cpp +utils/funcexp/func_truncate.cpp +utils/funcexp/func_ucase.cpp +utils/funcexp/func_unhex.cpp +utils/funcexp/func_unix_timestamp.cpp +utils/funcexp/func_week.cpp +utils/funcexp/func_weekday.cpp +utils/funcexp/func_year.cpp +utils/funcexp/func_yearweek.cpp +utils/funcexp/funcexp.cpp +utils/funcexp/funcexp.h +utils/funcexp/funcexpwrapper.cpp +utils/funcexp/funcexpwrapper.h +utils/funcexp/funchelpers.h +utils/funcexp/functor.cpp +utils/funcexp/functor.h +utils/funcexp/functor_all.h +utils/funcexp/functor_bool.h +utils/funcexp/functor_dtm.h +utils/funcexp/functor_export.h +utils/funcexp/functor_int.h +utils/funcexp/functor_real.h +utils/funcexp/functor_str.h +utils/funcexp/sha.h +utils/funcexp/tdriver.cpp +utils/funcexp/timeextract.h +utils/funcexp/utf8.h +utils/funcexp/utf8/checked.h +utils/funcexp/utf8/core.h +utils/funcexp/utf8/unchecked.h +utils/funcexp/utils_utf8.h +utils/idbdatafile/BufferedFile.cpp +utils/idbdatafile/BufferedFile.h +utils/idbdatafile/BufferedFileFactory.h +utils/idbdatafile/FileFactoryBase.h +utils/idbdatafile/IDBDataFile.cpp +utils/idbdatafile/IDBDataFile.h +utils/idbdatafile/IDBFactory.cpp +utils/idbdatafile/IDBFactory.h +utils/idbdatafile/IDBFileSystem.cpp +utils/idbdatafile/IDBFileSystem.h +utils/idbdatafile/IDBLogger.cpp +utils/idbdatafile/IDBLogger.h +utils/idbdatafile/IDBPolicy.cpp +utils/idbdatafile/IDBPolicy.h +utils/idbdatafile/PosixFileSystem.cpp +utils/idbdatafile/PosixFileSystem.h +utils/idbdatafile/UnbufferedFile.cpp +utils/idbdatafile/UnbufferedFile.h +utils/idbdatafile/UnbufferedFileFactory.h +utils/idbdatafile/largefile64.h +utils/idbdatafile/tdriver.cpp +utils/idbdatafile/tdriver1.cpp +utils/idbdatafile/utility.h +utils/idbhdfs/hdfs-12/hdfs.h +utils/idbhdfs/hdfs-20/hdfs.h +utils/idbhdfs/hdfs-shared/HdfsFile.cpp +utils/idbhdfs/hdfs-shared/HdfsFile.h +utils/idbhdfs/hdfs-shared/HdfsFileFactory.h +utils/idbhdfs/hdfs-shared/HdfsFileSystem.cpp +utils/idbhdfs/hdfs-shared/HdfsFileSystem.h +utils/idbhdfs/hdfs-shared/HdfsFsCache.cpp +utils/idbhdfs/hdfs-shared/HdfsFsCache.h +utils/idbhdfs/hdfs-shared/HdfsPlugin.cpp +utils/idbhdfs/hdfs-shared/HdfsRdwrFileBuffer.cpp +utils/idbhdfs/hdfs-shared/HdfsRdwrFileBuffer.h +utils/idbhdfs/hdfs-shared/HdfsRdwrMemBuffer.cpp +utils/idbhdfs/hdfs-shared/HdfsRdwrMemBuffer.h +utils/joiner/btree.h +utils/joiner/btree_container.h +utils/joiner/btree_map.h +utils/joiner/btree_set.h +utils/joiner/joiner.cpp +utils/joiner/joiner.h +utils/joiner/joinpartition.cpp +utils/joiner/joinpartition.h +utils/joiner/tuplejoiner.cpp +utils/joiner/tuplejoiner.h +utils/loggingcpp/errorcodes.cpp +utils/loggingcpp/errorcodes.h +utils/loggingcpp/exceptclasses.h +utils/loggingcpp/idberrorinfo.cpp +utils/loggingcpp/idberrorinfo.h +utils/loggingcpp/logger.cpp +utils/loggingcpp/logger.h +utils/loggingcpp/loggingid.h +utils/loggingcpp/message.cpp +utils/loggingcpp/messagelog.cpp +utils/loggingcpp/messagelog.h +utils/loggingcpp/messageobj.h +utils/loggingcpp/sqllogger.cpp +utils/loggingcpp/sqllogger.h +utils/loggingcpp/stopwatch.cpp +utils/loggingcpp/stopwatch.h +utils/loggingcpp/tdriver.cpp +utils/messageqcpp/bytestream.cpp +utils/messageqcpp/bytestream.h +utils/messageqcpp/cli.cpp +utils/messageqcpp/compressed_iss.cpp +utils/messageqcpp/compressed_iss.h +utils/messageqcpp/inetstreamsocket.cpp +utils/messageqcpp/inetstreamsocket.h +utils/messageqcpp/iosocket.cpp +utils/messageqcpp/iosocket.h +utils/messageqcpp/messagequeue.cpp +utils/messageqcpp/messagequeue.h +utils/messageqcpp/messagequeuepool.cpp +utils/messageqcpp/messagequeuepool.h +utils/messageqcpp/serializeable.h +utils/messageqcpp/serversocket.h +utils/messageqcpp/socket.h +utils/messageqcpp/socketclosed.h +utils/messageqcpp/socketparms.cpp +utils/messageqcpp/socketparms.h +utils/messageqcpp/srv.cpp +utils/messageqcpp/tdriver.cpp +utils/multicast/config.h +utils/multicast/impl.cpp +utils/multicast/impl.h +utils/multicast/mc_fifo.h +utils/multicast/multicast.cpp +utils/multicast/multicast.h +utils/multicast/participants.h +utils/multicast/produconsum.h +utils/multicast/socklib.h +utils/multicast/statistics.h +utils/multicast/threads.h +utils/multicast/udp-sender.h +utils/multicast/udpc-protoc.h +utils/multicast/udpcast.h +utils/multicast/util.h +utils/querystats/querystats.cpp +utils/querystats/querystats.h +utils/querytele/QueryTeleService.cpp +utils/querytele/QueryTeleService.h +utils/querytele/querystepparms.h +utils/querytele/querytele.cpp +utils/querytele/querytele.h +utils/querytele/querytele_constants.cpp +utils/querytele/querytele_constants.h +utils/querytele/querytele_types.cpp +utils/querytele/querytele_types.h +utils/querytele/queryteleclient.cpp +utils/querytele/queryteleclient.h +utils/querytele/queryteleprotoimpl.cpp +utils/querytele/queryteleprotoimpl.h +utils/querytele/queryteleserverparms.h +utils/querytele/serverdriver.cpp +utils/querytele/tdriver.cpp +utils/querytele/telestats.h +utils/rowgroup/rowaggregation.cpp +utils/rowgroup/rowaggregation.h +utils/rowgroup/rowgroup.cpp +utils/rowgroup/rowgroup.h +utils/rwlock/rwlock.cpp +utils/rwlock/rwlock.h +utils/rwlock/rwlock_local.cpp +utils/rwlock/rwlock_local.h +utils/rwlock/tdriver-rw.cpp +utils/rwlock/tdriver.cpp +utils/startup/installdir.cpp +utils/startup/installdir.h +utils/testbc/blockcacheclient.cpp +utils/testbc/blockcacheclient.h +utils/testbc/blockrequestprocessor.cpp +utils/testbc/blockrequestprocessor.h +utils/testbc/fileblockrequestqueue.cpp +utils/testbc/fileblockrequestqueue.h +utils/testbc/filebuffer.cpp +utils/testbc/filebuffer.h +utils/testbc/filebuffermgr.cpp +utils/testbc/filebuffermgr.h +utils/testbc/filerequest.cpp +utils/testbc/filerequest.h +utils/testbc/iomanager.cpp +utils/testbc/iomanager.h +utils/testbc/logger.cpp +utils/testbc/logger.h +utils/testbc/stats.cpp +utils/testbc/stats.h +utils/testbc/testbc.cpp +utils/testbc/testbc2.cpp +utils/testbc/testbc3.cpp +utils/threadpool/prioritythreadpool.cpp +utils/threadpool/prioritythreadpool.h +utils/threadpool/tdriver.cpp +utils/threadpool/threadpool.cpp +utils/threadpool/threadpool.h +utils/threadpool/tp.cpp +utils/threadpool/weightedthreadpool.cpp +utils/threadpool/weightedthreadpool.h +utils/threadpool/wtp.cpp +utils/thrift/thrift/TApplicationException.cpp +utils/thrift/thrift/TApplicationException.h +utils/thrift/thrift/TDispatchProcessor.h +utils/thrift/thrift/TLogging.h +utils/thrift/thrift/TProcessor.h +utils/thrift/thrift/TReflectionLocal.h +utils/thrift/thrift/Thrift.cpp +utils/thrift/thrift/Thrift.h +utils/thrift/thrift/async/TAsyncBufferProcessor.h +utils/thrift/thrift/async/TAsyncChannel.h +utils/thrift/thrift/async/TAsyncDispatchProcessor.h +utils/thrift/thrift/async/TAsyncProcessor.h +utils/thrift/thrift/async/TAsyncProtocolProcessor.h +utils/thrift/thrift/async/TEvhttpClientChannel.h +utils/thrift/thrift/async/TEvhttpServer.h +utils/thrift/thrift/concurrency/BoostThreadFactory.h +utils/thrift/thrift/concurrency/Exception.h +utils/thrift/thrift/concurrency/FunctionRunner.h +utils/thrift/thrift/concurrency/Monitor.h +utils/thrift/thrift/concurrency/Mutex.h +utils/thrift/thrift/concurrency/PlatformThreadFactory.h +utils/thrift/thrift/concurrency/PosixThreadFactory.h +utils/thrift/thrift/concurrency/StdThreadFactory.h +utils/thrift/thrift/concurrency/Thread.h +utils/thrift/thrift/concurrency/ThreadManager.h +utils/thrift/thrift/concurrency/TimerManager.h +utils/thrift/thrift/concurrency/Util.h +utils/thrift/thrift/config.h +utils/thrift/thrift/cxxfunctional.h +utils/thrift/thrift/processor/PeekProcessor.h +utils/thrift/thrift/processor/StatsProcessor.h +utils/thrift/thrift/processor/TMultiplexedProcessor.h +utils/thrift/thrift/protocol/TBase64Utils.h +utils/thrift/thrift/protocol/TBinaryProtocol.h +utils/thrift/thrift/protocol/TCompactProtocol.h +utils/thrift/thrift/protocol/TDebugProtocol.h +utils/thrift/thrift/protocol/TDenseProtocol.h +utils/thrift/thrift/protocol/TJSONProtocol.h +utils/thrift/thrift/protocol/TMultiplexedProtocol.h +utils/thrift/thrift/protocol/TProtocol.h +utils/thrift/thrift/protocol/TProtocolDecorator.h +utils/thrift/thrift/protocol/TProtocolException.h +utils/thrift/thrift/protocol/TProtocolTap.h +utils/thrift/thrift/protocol/TVirtualProtocol.h +utils/thrift/thrift/qt/TQIODeviceTransport.h +utils/thrift/thrift/qt/TQTcpServer.h +utils/thrift/thrift/server/TNonblockingServer.h +utils/thrift/thrift/server/TServer.h +utils/thrift/thrift/server/TSimpleServer.cpp +utils/thrift/thrift/server/TSimpleServer.h +utils/thrift/thrift/server/TThreadPoolServer.h +utils/thrift/thrift/server/TThreadedServer.h +utils/thrift/thrift/thrift-config.h +utils/thrift/thrift/transport/PlatformSocket.h +utils/thrift/thrift/transport/TBufferTransports.cpp +utils/thrift/thrift/transport/TBufferTransports.h +utils/thrift/thrift/transport/TFDTransport.h +utils/thrift/thrift/transport/TFileTransport.h +utils/thrift/thrift/transport/THttpClient.h +utils/thrift/thrift/transport/THttpServer.h +utils/thrift/thrift/transport/THttpTransport.h +utils/thrift/thrift/transport/TPipe.h +utils/thrift/thrift/transport/TPipeServer.h +utils/thrift/thrift/transport/TSSLServerSocket.h +utils/thrift/thrift/transport/TSSLSocket.h +utils/thrift/thrift/transport/TServerSocket.cpp +utils/thrift/thrift/transport/TServerSocket.h +utils/thrift/thrift/transport/TServerTransport.h +utils/thrift/thrift/transport/TShortReadTransport.h +utils/thrift/thrift/transport/TSimpleFileTransport.h +utils/thrift/thrift/transport/TSocket.cpp +utils/thrift/thrift/transport/TSocket.h +utils/thrift/thrift/transport/TSocketPool.h +utils/thrift/thrift/transport/TTransport.h +utils/thrift/thrift/transport/TTransportException.h +utils/thrift/thrift/transport/TTransportUtils.h +utils/thrift/thrift/transport/TVirtualTransport.h +utils/thrift/thrift/transport/TZlibTransport.h +utils/thrift/thrift/windows/GetTimeOfDay.cpp +utils/thrift/thrift/windows/GetTimeOfDay.h +utils/thrift/thrift/windows/Operators.h +utils/thrift/thrift/windows/SocketPair.h +utils/thrift/thrift/windows/StdAfx.h +utils/thrift/thrift/windows/TWinsockSingleton.cpp +utils/thrift/thrift/windows/TWinsockSingleton.h +utils/thrift/thrift/windows/TargetVersion.h +utils/thrift/thrift/windows/WinFcntl.cpp +utils/thrift/thrift/windows/WinFcntl.h +utils/thrift/thrift/windows/config.h +utils/thrift/thrift/windows/force_inc.h +utils/udfsdk/resource.h +utils/udfsdk/udfinfinidb.cpp +utils/udfsdk/udfmysql.cpp +utils/udfsdk/udfsdk.h +utils/windowfunction/framebound.cpp +utils/windowfunction/framebound.h +utils/windowfunction/frameboundrange.cpp +utils/windowfunction/frameboundrange.h +utils/windowfunction/frameboundrow.cpp +utils/windowfunction/frameboundrow.h +utils/windowfunction/idborderby.cpp +utils/windowfunction/idborderby.h +utils/windowfunction/wf_count.cpp +utils/windowfunction/wf_count.h +utils/windowfunction/wf_lead_lag.cpp +utils/windowfunction/wf_lead_lag.h +utils/windowfunction/wf_min_max.cpp +utils/windowfunction/wf_min_max.h +utils/windowfunction/wf_nth_value.cpp +utils/windowfunction/wf_nth_value.h +utils/windowfunction/wf_ntile.cpp +utils/windowfunction/wf_ntile.h +utils/windowfunction/wf_percentile.cpp +utils/windowfunction/wf_percentile.h +utils/windowfunction/wf_ranking.cpp +utils/windowfunction/wf_ranking.h +utils/windowfunction/wf_row_number.cpp +utils/windowfunction/wf_row_number.h +utils/windowfunction/wf_stats.cpp +utils/windowfunction/wf_stats.h +utils/windowfunction/wf_sum_avg.cpp +utils/windowfunction/wf_sum_avg.h +utils/windowfunction/windowframe.cpp +utils/windowfunction/windowframe.h +utils/windowfunction/windowfunction.cpp +utils/windowfunction/windowfunction.h +utils/windowfunction/windowfunctiontype.cpp +utils/windowfunction/windowfunctiontype.h +utils/winport/WinSyslog.cpp +utils/winport/WinSyslog.h +utils/winport/afxres.h +utils/winport/bootstrap.cpp +utils/winport/crc.c +utils/winport/fixup.cpp +utils/winport/fixup.h +utils/winport/glob.c +utils/winport/glob.h +utils/winport/grepit.cpp +utils/winport/grepit.h +utils/winport/idb_getopt.cpp +utils/winport/idb_getopt.h +utils/winport/idbregistry.cpp +utils/winport/idbregistry.h +utils/winport/inttypes.h +utils/winport/main.cpp +utils/winport/poll.c +utils/winport/poll.h +utils/winport/resource.h +utils/winport/sedit.cpp +utils/winport/sedit.h +utils/winport/strtoll.c +utils/winport/sys/time.h +utils/winport/sysinfo.cpp +utils/winport/sysinfo.h +utils/winport/syslog.h +utils/winport/unistd.h +utils/winport/winfinidb.cpp +utils/winport/winport.cpp +utils/winport/winport.h +versioning/BRM/autoincrementmanager.cpp +versioning/BRM/autoincrementmanager.h +versioning/BRM/autoinctest.cpp +versioning/BRM/blockresolutionmanager.cpp +versioning/BRM/blockresolutionmanager.h +versioning/BRM/brm.h +versioning/BRM/brmshmimpl.cpp +versioning/BRM/brmshmimpl.h +versioning/BRM/brmtypes.cpp +versioning/BRM/brmtypes.h +versioning/BRM/copylocks.cpp +versioning/BRM/copylocks.h +versioning/BRM/copylocktest.cpp +versioning/BRM/cvt_em.cpp +versioning/BRM/dbrm.cpp +versioning/BRM/dbrm.h +versioning/BRM/dbrmctl.cpp +versioning/BRM/extentmap.cpp +versioning/BRM/extentmap.h +versioning/BRM/lbidresourcegraph.cpp +versioning/BRM/lbidresourcegraph.h +versioning/BRM/load_brm.cpp +versioning/BRM/load_brm64.cpp +versioning/BRM/load_brm_from_file.cpp +versioning/BRM/load_em.cpp +versioning/BRM/lock_grabber.cpp +versioning/BRM/lock_state.cpp +versioning/BRM/logicalpartition.cpp +versioning/BRM/logicalpartition.h +versioning/BRM/masterdbrmnode.cpp +versioning/BRM/masterdbrmnode.h +versioning/BRM/masternode.cpp +versioning/BRM/mastersegmenttable.cpp +versioning/BRM/mastersegmenttable.h +versioning/BRM/oidserver.cpp +versioning/BRM/oidserver.h +versioning/BRM/reset_locks.cpp +versioning/BRM/resource.h +versioning/BRM/resourcenode.cpp +versioning/BRM/resourcenode.h +versioning/BRM/rgnode.cpp +versioning/BRM/rgnode.h +versioning/BRM/rollback.cpp +versioning/BRM/rwlockmonitor.cpp +versioning/BRM/rwlockmonitor.h +versioning/BRM/save_brm.cpp +versioning/BRM/sessionmanagerserver.cpp +versioning/BRM/sessionmanagerserver.h +versioning/BRM/shmkeys.cpp +versioning/BRM/shmkeys.h +versioning/BRM/slavecomm.cpp +versioning/BRM/slavecomm.h +versioning/BRM/slavedbrmnode.cpp +versioning/BRM/slavedbrmnode.h +versioning/BRM/slavenode.cpp +versioning/BRM/tablelockserver.cpp +versioning/BRM/tablelockserver.h +versioning/BRM/tablelocktest.cpp +versioning/BRM/tdriver-brm.cpp +versioning/BRM/tdriver-dbrm.cpp +versioning/BRM/tdriver-dbrm2.cpp +versioning/BRM/tdriver-load.cpp +versioning/BRM/tdriver.cpp +versioning/BRM/tracer.cpp +versioning/BRM/tracer.h +versioning/BRM/transactionnode.cpp +versioning/BRM/transactionnode.h +versioning/BRM/undoable.cpp +versioning/BRM/undoable.h +versioning/BRM/vbbm.cpp +versioning/BRM/vbbm.h +versioning/BRM/vss.cpp +versioning/BRM/vss.h +versioning/CMakeFiles/3.8.2/CompilerIdC/CMakeCCompilerId.c +versioning/CMakeFiles/3.8.2/CompilerIdCXX/CMakeCXXCompilerId.cpp +versioning/CMakeFiles/feature_tests.c +versioning/CMakeFiles/feature_tests.cxx +writeengine/bulk/cpimport.cpp +writeengine/bulk/parsetest.cpp +writeengine/bulk/resource.h +writeengine/bulk/we_brmreporter.cpp +writeengine/bulk/we_brmreporter.h +writeengine/bulk/we_bulkload.cpp +writeengine/bulk/we_bulkload.h +writeengine/bulk/we_bulkloadbuffer.cpp +writeengine/bulk/we_bulkloadbuffer.h +writeengine/bulk/we_bulkstatus.cpp +writeengine/bulk/we_bulkstatus.h +writeengine/bulk/we_colbuf.cpp +writeengine/bulk/we_colbuf.h +writeengine/bulk/we_colbufcompressed.cpp +writeengine/bulk/we_colbufcompressed.h +writeengine/bulk/we_colbufmgr.cpp +writeengine/bulk/we_colbufmgr.h +writeengine/bulk/we_colbufmgrdctnry.cpp +writeengine/bulk/we_colbufsec.cpp +writeengine/bulk/we_colbufsec.h +writeengine/bulk/we_colextinf.cpp +writeengine/bulk/we_colextinf.h +writeengine/bulk/we_colopbulk.cpp +writeengine/bulk/we_colopbulk.h +writeengine/bulk/we_columnautoinc.cpp +writeengine/bulk/we_columnautoinc.h +writeengine/bulk/we_columninfo.cpp +writeengine/bulk/we_columninfo.h +writeengine/bulk/we_columninfocompressed.cpp +writeengine/bulk/we_columninfocompressed.h +writeengine/bulk/we_extentstripealloc.cpp +writeengine/bulk/we_extentstripealloc.h +writeengine/bulk/we_tableinfo.cpp +writeengine/bulk/we_tableinfo.h +writeengine/bulk/we_tempxmlgendata.cpp +writeengine/bulk/we_tempxmlgendata.h +writeengine/bulk/we_workers.cpp +writeengine/client/we_clients.cpp +writeengine/client/we_clients.h +writeengine/client/we_ddlcommandclient.cpp +writeengine/client/we_ddlcommandclient.h +writeengine/client/we_dmlcommandclient.cpp +writeengine/client/we_dmlcommandclient.h +writeengine/dictionary/qdctnry.cpp +writeengine/dictionary/tdctnry.cpp +writeengine/dictionary/we_dctnry.cpp +writeengine/dictionary/we_dctnry.h +writeengine/dictionary/we_dctnrystore.cpp +writeengine/dictionary/we_dctnrystore.h +writeengine/index/qindex.cpp +writeengine/index/tindex.cpp +writeengine/index/we_freemgr.cpp +writeengine/index/we_freemgr.h +writeengine/index/we_indexlist.cpp +writeengine/index/we_indexlist.h +writeengine/index/we_indexlist_common.cpp +writeengine/index/we_indexlist_find_delete.cpp +writeengine/index/we_indexlist_multiple_narray.cpp +writeengine/index/we_indexlist_narray.cpp +writeengine/index/we_indexlist_update_hdr_sub.cpp +writeengine/index/we_indextree.cpp +writeengine/index/we_indextree.h +writeengine/redistribute/we_redistribute.cpp +writeengine/redistribute/we_redistribute.h +writeengine/redistribute/we_redistributecontrol.cpp +writeengine/redistribute/we_redistributecontrol.h +writeengine/redistribute/we_redistributecontrolthread.cpp +writeengine/redistribute/we_redistributecontrolthread.h +writeengine/redistribute/we_redistributedef.h +writeengine/redistribute/we_redistributeworkerthread.cpp +writeengine/redistribute/we_redistributeworkerthread.h +writeengine/resource.h +writeengine/server/resource.h +writeengine/server/we_brmrprtparser.cpp +writeengine/server/we_brmrprtparser.h +writeengine/server/we_cleartablelockcmd.cpp +writeengine/server/we_cleartablelockcmd.h +writeengine/server/we_cpifeederthread.cpp +writeengine/server/we_cpifeederthread.h +writeengine/server/we_dataloader.cpp +writeengine/server/we_dataloader.h +writeengine/server/we_ddlcommandproc.cpp +writeengine/server/we_ddlcommandproc.h +writeengine/server/we_ddlcommon.h +writeengine/server/we_dmlcommandproc.cpp +writeengine/server/we_dmlcommandproc.h +writeengine/server/we_getfilesizes.cpp +writeengine/server/we_getfilesizes.h +writeengine/server/we_message_handlers.h +writeengine/server/we_messages.h +writeengine/server/we_msg1.cpp +writeengine/server/we_msg2.cpp +writeengine/server/we_observer.cpp +writeengine/server/we_observer.h +writeengine/server/we_readthread.cpp +writeengine/server/we_readthread.h +writeengine/server/we_server.cpp +writeengine/shared/tconfig.cpp +writeengine/shared/tshared.cpp +writeengine/shared/we_blockop.cpp +writeengine/shared/we_blockop.h +writeengine/shared/we_brm.cpp +writeengine/shared/we_brm.h +writeengine/shared/we_bulkrollbackfile.cpp +writeengine/shared/we_bulkrollbackfile.h +writeengine/shared/we_bulkrollbackfilecompressed.cpp +writeengine/shared/we_bulkrollbackfilecompressed.h +writeengine/shared/we_bulkrollbackfilecompressedhdfs.cpp +writeengine/shared/we_bulkrollbackfilecompressedhdfs.h +writeengine/shared/we_bulkrollbackmgr.cpp +writeengine/shared/we_bulkrollbackmgr.h +writeengine/shared/we_cache.cpp +writeengine/shared/we_cache.h +writeengine/shared/we_chunkmanager.cpp +writeengine/shared/we_chunkmanager.h +writeengine/shared/we_config.cpp +writeengine/shared/we_config.h +writeengine/shared/we_confirmhdfsdbfile.cpp +writeengine/shared/we_confirmhdfsdbfile.h +writeengine/shared/we_convertor.cpp +writeengine/shared/we_convertor.h +writeengine/shared/we_dbfileop.cpp +writeengine/shared/we_dbfileop.h +writeengine/shared/we_dbrootextenttracker.cpp +writeengine/shared/we_dbrootextenttracker.h +writeengine/shared/we_define.cpp +writeengine/shared/we_define.h +writeengine/shared/we_fileop.cpp +writeengine/shared/we_fileop.h +writeengine/shared/we_index.h +writeengine/shared/we_log.cpp +writeengine/shared/we_log.h +writeengine/shared/we_macro.h +writeengine/shared/we_obj.h +writeengine/shared/we_rbmetawriter.cpp +writeengine/shared/we_rbmetawriter.h +writeengine/shared/we_simplesyslog.cpp +writeengine/shared/we_simplesyslog.h +writeengine/shared/we_stats.cpp +writeengine/shared/we_stats.h +writeengine/shared/we_type.h +writeengine/shared/we_typeext.h +writeengine/splitter/resource.h +writeengine/splitter/we_brmupdater.cpp +writeengine/splitter/we_brmupdater.h +writeengine/splitter/we_cmdargs.cpp +writeengine/splitter/we_cmdargs.h +writeengine/splitter/we_filereadthread.cpp +writeengine/splitter/we_filereadthread.h +writeengine/splitter/we_respreadthread.cpp +writeengine/splitter/we_respreadthread.h +writeengine/splitter/we_sdhandler.cpp +writeengine/splitter/we_sdhandler.h +writeengine/splitter/we_splclient.cpp +writeengine/splitter/we_splclient.h +writeengine/splitter/we_splitterapp.cpp +writeengine/splitter/we_splitterapp.h +writeengine/splitter/we_tablelockgrabber.cpp +writeengine/splitter/we_tablelockgrabber.h +writeengine/splitter/we_xmlgetter.cpp +writeengine/splitter/we_xmlgetter.h +writeengine/wrapper/tdriver.cpp +writeengine/wrapper/we_colop.cpp +writeengine/wrapper/we_colop.h +writeengine/wrapper/we_colopcompress.cpp +writeengine/wrapper/we_colopcompress.h +writeengine/wrapper/we_dctnrycompress.cpp +writeengine/wrapper/we_dctnrycompress.h +writeengine/wrapper/we_tablemetadata.cpp +writeengine/wrapper/we_tablemetadata.h +writeengine/wrapper/writeengine.cpp +writeengine/wrapper/writeengine.h +writeengine/xml/txml.cpp +writeengine/xml/we_xmlgendata.cpp +writeengine/xml/we_xmlgendata.h +writeengine/xml/we_xmlgenproc.cpp +writeengine/xml/we_xmlgenproc.h +writeengine/xml/we_xmljob.cpp +writeengine/xml/we_xmljob.h +writeengine/xml/we_xmlop.cpp +writeengine/xml/we_xmlop.h +writeengine/xml/we_xmltag.h diff --git a/mariadb-columnstore-engine.includes b/mariadb-columnstore-engine.includes new file mode 100644 index 000000000..7cfcd1dc8 --- /dev/null +++ b/mariadb-columnstore-engine.includes @@ -0,0 +1,81 @@ +dbcon/ddlpackage +dbcon/ddlpackageproc +dbcon/dmlpackage +dbcon/dmlpackageproc +dbcon/doc +dbcon/execplan +dbcon/joblist +dbcon/mysql +ddlproc +decomsvr +dmlproc +exemgr +oam/oamcpp +oam/replaytxnlog +oamapps/alarmmanager +oamapps/calpontConsole +oamapps/hardwareMonitor +oamapps/mcsadmin +oamapps/postConfigure +oamapps/serverMonitor +primitives/blockcache +primitives/linux-port +primitives/primproc +procmgr +procmon +tools/cleartablelock +tools/dbbuilder +tools/dbloadxml +tools/dmldriver +tools/qfe +tools/vbgen +utils/batchloader +utils/cacheutils +utils/common +utils/compress +utils/configcpp +utils/configcpp/md5 +utils/dataconvert +utils/ddlcleanup +utils/funcexp +utils/funcexp/utf8 +utils/idbdatafile +utils/idbhdfs/hdfs-12 +utils/idbhdfs/hdfs-20 +utils/idbhdfs/hdfs-shared +utils/joiner +utils/loggingcpp +utils/messageqcpp +utils/multicast +utils/querystats +utils/querytele +utils/rowgroup +utils/rwlock +utils/startup +utils/testbc +utils/threadpool +utils/thrift/thrift +utils/thrift/thrift/async +utils/thrift/thrift/concurrency +utils/thrift/thrift/processor +utils/thrift/thrift/protocol +utils/thrift/thrift/qt +utils/thrift/thrift/server +utils/thrift/thrift/transport +utils/thrift/thrift/windows +utils/udfsdk +utils/windowfunction +utils/winport +utils/winport/sys +versioning/BRM +writeengine +writeengine/bulk +writeengine/client +writeengine/dictionary +writeengine/index +writeengine/redistribute +writeengine/server +writeengine/shared +writeengine/splitter +writeengine/wrapper +writeengine/xml diff --git a/writeengine/server/we_dmlcommandproc.cpp b/writeengine/server/we_dmlcommandproc.cpp index 26b2340ad..c57faa0e3 100644 --- a/writeengine/server/we_dmlcommandproc.cpp +++ b/writeengine/server/we_dmlcommandproc.cpp @@ -1292,7 +1292,7 @@ uint8_t WE_DMLCommandProc::processBatchInsertBinary(messageqcpp::ByteStream& bs, WriteEngine::ColStructList colStructs; WriteEngine::DctnryStructList dctnryStructList; WriteEngine::DctnryValueList dctnryValueList; - WriteEngine::ColValueList colValuesList; + std::vector colValuesList; WriteEngine::DictStrList dicStringList ; CalpontSystemCatalog::TableName tableName; CalpontSystemCatalog::TableColName tableColName; @@ -1520,12 +1520,12 @@ uint8_t WE_DMLCommandProc::processBatchInsertBinary(messageqcpp::ByteStream& bs, std::string tmpStr(""); uint32_t valuesPerColumn; bs >> valuesPerColumn; + colValuesList.reserve(columnCount * valuesPerColumn); try { bool pushWarning = false; for (uint32_t j = 0; j < columnCount; j++) { - WriteEngine::ColTupleList colTuples; WriteEngine::DctColTupleList dctColTuples; tableColName.column = colNames[j]; CalpontSystemCatalog::OID oid = colStructs[j].dataOid; @@ -1533,7 +1533,6 @@ uint8_t WE_DMLCommandProc::processBatchInsertBinary(messageqcpp::ByteStream& bs, CalpontSystemCatalog::ColType colType; colType = systemCatalogPtr->colType(oid); - boost::any datavalue; bool isNULL = false; WriteEngine::dictStr dicStrings; // token @@ -1569,14 +1568,10 @@ uint8_t WE_DMLCommandProc::processBatchInsertBinary(messageqcpp::ByteStream& bs, if ( !pushWarning ) pushWarning = true; } - WriteEngine::ColTuple colTuple; - colTuple.data = datavalue; - - colTuples.push_back(colTuple); + colValuesList.push_back(0); //@Bug 2515. Only pass string values to write engine dicStrings.push_back( tmpStr ); } - colValuesList.push_back(colTuples); //@Bug 2515. Only pass string values to write engine dicStringList.push_back( dicStrings ); } @@ -1606,67 +1601,48 @@ uint8_t WE_DMLCommandProc::processBatchInsertBinary(messageqcpp::ByteStream& bs, bs >> tmp8; isNULL = tmp8; - int8_t val8; - int16_t val16; - int32_t val32; - int64_t val64; + uint8_t val8; + uint16_t val16; + uint32_t val32; + uint64_t val64; + uint64_t colValue; float valF; double valD; std::string valStr; bool valZero = false; // Needed for autoinc check - switch (colType.colDataType) + switch (colType.colDataType) { case execplan::CalpontSystemCatalog::TINYINT: - bs >> val8; + case execplan::CalpontSystemCatalog::UTINYINT: + bs >> val8; if (val8 == 0) valZero = true; - datavalue = (char)val8; + colValue = val8; break; case execplan::CalpontSystemCatalog::SMALLINT: - bs >> val16; + case execplan::CalpontSystemCatalog::DATE: + case execplan::CalpontSystemCatalog::USMALLINT: + bs >> val16; if (val16 == 0) valZero = true; - datavalue = (short)val16; + colValue = val16; break; case execplan::CalpontSystemCatalog::MEDINT: case execplan::CalpontSystemCatalog::INT: - bs >> val32; + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UINT: + bs >> val32; if (val32 == 0) valZero = true; - datavalue = (int)val32; + colValue = val32; break; case execplan::CalpontSystemCatalog::BIGINT: - bs >> val64; + case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::UBIGINT: + bs >> val64; if (val64 == 0) valZero = true; - datavalue = (long long)val64; - break; - case execplan::CalpontSystemCatalog::UTINYINT: - bs >> val8; - if (val8 == 0) - valZero = true; - datavalue = (uint8_t)val8; - break; - case execplan::CalpontSystemCatalog::DATE: - case execplan::CalpontSystemCatalog::USMALLINT: - bs >> val16; - if (val16 == 0) - valZero = true; - datavalue = (uint16_t)val16; - break; - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UINT: - bs >> val32; - if (val32 == 0) - valZero = true; - datavalue = (uint32_t)val32; - break; - case execplan::CalpontSystemCatalog::DATETIME: - case execplan::CalpontSystemCatalog::UBIGINT: - bs >> val64; - if (val64 == 0) - valZero = true; - datavalue = (uint64_t)val64; + colValue = val64; break; case execplan::CalpontSystemCatalog::DECIMAL: switch (colType.colWidth) @@ -1674,25 +1650,25 @@ uint8_t WE_DMLCommandProc::processBatchInsertBinary(messageqcpp::ByteStream& bs, case 1: { bs >> val8; - datavalue = (char) val8; + colValue = val8; break; } case 2: { bs >> val16; - datavalue = (short) val16; + colValue = val16; break; } case 4: { bs >> val32; - datavalue = (int) val32; + colValue = val32; break; } default: { bs >> val64; - datavalue = (long long) val64; + colValue = val64; break; } } @@ -1710,7 +1686,7 @@ uint8_t WE_DMLCommandProc::processBatchInsertBinary(messageqcpp::ByteStream& bs, val8 = 0; pushWarning = true; } - datavalue = (char)val8; + colValue = val8; } else if (colType.colWidth == 2) { @@ -1722,7 +1698,7 @@ uint8_t WE_DMLCommandProc::processBatchInsertBinary(messageqcpp::ByteStream& bs, val16 = 0; pushWarning = true; } - datavalue = (short)val16; + colValue = val16; } else if (colType.colWidth == 4) { @@ -1734,7 +1710,7 @@ uint8_t WE_DMLCommandProc::processBatchInsertBinary(messageqcpp::ByteStream& bs, val32 = 0; pushWarning = true; } - datavalue = (int)val32; + colValue = val32; } else if (colType.colWidth == 8) { @@ -1746,13 +1722,12 @@ uint8_t WE_DMLCommandProc::processBatchInsertBinary(messageqcpp::ByteStream& bs, val64 = 0; pushWarning = true; } - datavalue = (long long)val64; + colValue = val64; } break; case execplan::CalpontSystemCatalog::DOUBLE: bs >> val64; - memcpy(&valD, &val64, 8); - datavalue = valD; + colValue = val64; break; case execplan::CalpontSystemCatalog::UDOUBLE: bs >> val64; @@ -1763,12 +1738,11 @@ uint8_t WE_DMLCommandProc::processBatchInsertBinary(messageqcpp::ByteStream& bs, pushWarning = true; } - datavalue = valD; + colValue = val64; break; case execplan::CalpontSystemCatalog::FLOAT: bs >> val32; - memcpy(&valF, &val32, 4); - datavalue = valF; + colValue = val32; break; case execplan::CalpontSystemCatalog::UFLOAT: bs >> val32; @@ -1779,7 +1753,7 @@ uint8_t WE_DMLCommandProc::processBatchInsertBinary(messageqcpp::ByteStream& bs, pushWarning = true; } - datavalue = valF; + colValue = val32; break; case execplan::CalpontSystemCatalog::CHAR: @@ -1800,7 +1774,7 @@ uint8_t WE_DMLCommandProc::processBatchInsertBinary(messageqcpp::ByteStream& bs, valStr.resize(colType.colWidth, 0); } } - datavalue = valStr; + memcpy(&colValue, valStr.c_str(), valStr.length()); break; default: rc = 1; @@ -1830,28 +1804,7 @@ uint8_t WE_DMLCommandProc::processBatchInsertBinary(messageqcpp::ByteStream& bs, rc = 1; return rc; } - switch (colType.colDataType) - { - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::UTINYINT: - datavalue = (uint8_t) nextVal; - break; - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::USMALLINT: - datavalue = (uint16_t) nextVal; - break; - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::UINT: - datavalue = (uint32_t) nextVal; - break; - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::UBIGINT: - default: - datavalue = (uint64_t) nextVal; - break; - } + colValue = nextVal; } if (colType.constraintType == CalpontSystemCatalog::NOTNULL_CONSTRAINT) @@ -1866,7 +1819,7 @@ uint8_t WE_DMLCommandProc::processBatchInsertBinary(messageqcpp::ByteStream& bs, } else if (isNULL && !(colType.defaultValue.empty())) { - datavalue = colType.defaultValue; + memcpy(&colValue, colType.defaultValue.c_str(), colType.defaultValue.length()); isNULL = false; } } @@ -1878,14 +1831,11 @@ uint8_t WE_DMLCommandProc::processBatchInsertBinary(messageqcpp::ByteStream& bs, if ( pushWarning && ( rc != dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING ) ) rc = dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING; - WriteEngine::ColTuple colTuple; - colTuple.data = datavalue; - colTuples.push_back(colTuple); + colValuesList.push_back(colValue); //@Bug 2515. Only pass string values to write engine dicStrings.push_back( valStr ); } - colValuesList.push_back(colTuples); dicStringList.push_back( dicStrings ); } @@ -1910,31 +1860,28 @@ uint8_t WE_DMLCommandProc::processBatchInsertBinary(messageqcpp::ByteStream& bs, //cout << "Batch inserting a row with transaction id " << txnid.id << endl; if (colValuesList.size() > 0) { - if (colValuesList[0].size() > 0) - { - if (NO_ERROR != - (error = fWEWrapper.insertColumnRecs(txnid.id, colStructs, colValuesList, dctnryStructList, dicStringList, - dbRootExtTrackerVec, 0, bFirstExtentOnThisPM, isInsertSelect, isAutocommitOn, roPair.objnum, fIsFirstBatchPm))) - { - if (error == ERR_BRM_DEAD_LOCK) - { - rc = dmlpackageprocessor::DMLPackageProcessor::DEAD_LOCK_ERROR; - WErrorCodes ec; - err = ec.errorString(error); - } - else if ( error == ERR_BRM_VB_OVERFLOW ) - { - rc = dmlpackageprocessor::DMLPackageProcessor::VB_OVERFLOW_ERROR; - err = IDBErrorInfo::instance()->errorMsg(ERR_VERSIONBUFFER_OVERFLOW); - } - else - { - rc = dmlpackageprocessor::DMLPackageProcessor::INSERT_ERROR; - WErrorCodes ec; - err = ec.errorString(error); - } - } - } + if (NO_ERROR != + (error = fWEWrapper.insertColumnRecsBinary(txnid.id, colStructs, colValuesList, dctnryStructList, dicStringList, + dbRootExtTrackerVec, 0, bFirstExtentOnThisPM, isInsertSelect, isAutocommitOn, roPair.objnum, fIsFirstBatchPm))) + { + if (error == ERR_BRM_DEAD_LOCK) + { + rc = dmlpackageprocessor::DMLPackageProcessor::DEAD_LOCK_ERROR; + WErrorCodes ec; + err = ec.errorString(error); + } + else if ( error == ERR_BRM_VB_OVERFLOW ) + { + rc = dmlpackageprocessor::DMLPackageProcessor::VB_OVERFLOW_ERROR; + err = IDBErrorInfo::instance()->errorMsg(ERR_VERSIONBUFFER_OVERFLOW); + } + else + { + rc = dmlpackageprocessor::DMLPackageProcessor::INSERT_ERROR; + WErrorCodes ec; + err = ec.errorString(error); + } + } } if (fIsFirstBatchPm && isAutocommitOn) { diff --git a/writeengine/wrapper/we_colop.h b/writeengine/wrapper/we_colop.h index 3bdb3bd8e..f9ff717a6 100644 --- a/writeengine/wrapper/we_colop.h +++ b/writeengine/wrapper/we_colop.h @@ -313,13 +313,13 @@ public: */ int copyVB(IDBDataFile* pSource, const BRM::VER_t txnD, const OID oid, std::vector& fboList, std::vector& rangeList); -protected: - /** * @brief close column file */ EXPORT virtual void closeColumnFile(Column& column) const; +protected: + /** * @brief populate readBuf with data in block #lbid */ diff --git a/writeengine/wrapper/writeengine.cpp b/writeengine/wrapper/writeengine.cpp index 727bdaf48..043490603 100644 --- a/writeengine/wrapper/writeengine.cpp +++ b/writeengine/wrapper/writeengine.cpp @@ -1457,6 +1457,638 @@ timer.start("writeColumnRec"); return rc; } +int WriteEngineWrapper::insertColumnRecsBinary(const TxnID& txnid, + ColStructList& colStructList, + std::vector& colValueList, + DctnryStructList& dctnryStructList, + DictStrList& dictStrList, + std::vector > & dbRootExtentTrackers, + RBMetaWriter* fRBMetaWriter, + bool bFirstExtentOnThisPM, + bool insertSelect, + bool isAutoCommitOn, + OID tableOid, + bool isFirstBatchPm) +{ + int rc; + RID* rowIdArray = NULL; + Column curCol; + ColStruct curColStruct; + ColStructList newColStructList; + DctnryStructList newDctnryStructList; + HWM hwm = 0; + HWM oldHwm = 0; + HWM newHwm = 0; + size_t totalRow; + ColStructList::size_type totalColumns; + uint64_t rowsLeft = 0; + bool newExtent = false; + RIDList ridList; + ColumnOp* colOp = NULL; + + // Set tmp file suffix to modify HDFS db file + bool useTmpSuffix = false; + if (idbdatafile::IDBPolicy::useHdfs()) + { + if (!bFirstExtentOnThisPM) + useTmpSuffix = true; + } + + unsigned i=0; +#ifdef PROFILE + StopWatch timer; +#endif + + //Convert data type and column width to write engine specific + for (i = 0; i < colStructList.size(); i++) + Convertor::convertColType(&colStructList[i]); + + // rc = checkValid(txnid, colStructList, colValueList, ridList); + // if (rc != NO_ERROR) + // return rc; + + setTransId(txnid); + uint16_t dbRoot, segmentNum; + uint32_t partitionNum; + string segFile; + bool newFile; + TableMetaData* tableMetaData= TableMetaData::makeTableMetaData(tableOid); + //populate colStructList with file information + IDBDataFile* pFile = NULL; + std::vector extentInfo; + int currentDBrootIdx = 0; + std::vector extents; + + //-------------------------------------------------------------------------- + // For first batch on this PM: + // o get starting extent from ExtentTracker, and allocate extent if needed + // o construct colStructList and dctnryStructList accordingly + // o save extent information in tableMetaData for future use + // If not first batch on this PM: + // o construct colStructList and dctnryStructList from tableMetaData + //-------------------------------------------------------------------------- + if (isFirstBatchPm) + { + currentDBrootIdx = dbRootExtentTrackers[0]->getCurrentDBRootIdx(); + extentInfo = dbRootExtentTrackers[0]->getDBRootExtentList(); + dbRoot = extentInfo[currentDBrootIdx].fDbRoot; + partitionNum = extentInfo[currentDBrootIdx].fPartition; + + //---------------------------------------------------------------------- + // check whether this extent is the first on this PM + //---------------------------------------------------------------------- + if (bFirstExtentOnThisPM) + { + //cout << "bFirstExtentOnThisPM is " << bFirstExtentOnThisPM << endl; + std::vector cols; + BRM::CreateStripeColumnExtentsArgIn createStripeColumnExtentsArgIn; + for (i=0; i < colStructList.size(); i++) + { + createStripeColumnExtentsArgIn.oid = colStructList[i].dataOid; + createStripeColumnExtentsArgIn.width = colStructList[i].colWidth; + createStripeColumnExtentsArgIn.colDataType = colStructList[i].colDataType; + cols.push_back(createStripeColumnExtentsArgIn); + } + rc = BRMWrapper::getInstance()->allocateStripeColExtents(cols, dbRoot, partitionNum, segmentNum, extents); + if (rc != NO_ERROR) + return rc; + //Create column files + BRM::CPInfoList_t cpinfoList; + BRM::CPInfo cpInfo; + if (isUnsigned(colStructList[i].colDataType)) + { + cpInfo.max = 0; + cpInfo.min = static_cast(numeric_limits::max()); + } + else + { + cpInfo.max = numeric_limits::min(); + cpInfo.min = numeric_limits::max(); + } + cpInfo.seqNum = -1; + for ( i=0; i < extents.size(); i++) + { + colOp = m_colOp[op(colStructList[i].fCompressionType)]; + colOp->initColumn(curCol); + colOp->setColParam(curCol, 0, colStructList[i].colWidth, colStructList[i].colDataType, + colStructList[i].colType, colStructList[i].dataOid, colStructList[i].fCompressionType, + dbRoot, partitionNum, segmentNum); + rc = colOp->extendColumn(curCol, false, extents[i].startBlkOffset, extents[i].startLbid, extents[i].allocSize, dbRoot, + partitionNum, segmentNum, segFile, pFile, newFile); + if (rc != NO_ERROR) + return rc; + + //mark the extents to invalid + cpInfo.firstLbid = extents[i].startLbid; + cpinfoList.push_back(cpInfo); + colStructList[i].fColPartition = partitionNum; + colStructList[i].fColSegment = segmentNum; + colStructList[i].fColDbRoot = dbRoot; + dctnryStructList[i].fColPartition = partitionNum; + dctnryStructList[i].fColSegment = segmentNum; + dctnryStructList[i].fColDbRoot = dbRoot; + } + + //mark the extents to invalid + rc = BRMWrapper::getInstance()->setExtentsMaxMin(cpinfoList); + if (rc != NO_ERROR) + return rc; + //create corresponding dictionary files + for (i=0; i < dctnryStructList.size(); i++) + { + if (dctnryStructList[i].dctnryOid > 0) + { + rc = createDctnry(txnid, dctnryStructList[i].dctnryOid, dctnryStructList[i].colWidth, dbRoot, partitionNum, + segmentNum, dctnryStructList[i].fCompressionType); + if ( rc != NO_ERROR) + return rc; + } + } + } // if ( bFirstExtentOnThisPM) + else // if (!bFirstExtentOnThisPM) + { + std::vector tmpExtentInfo; + for (i=0; i < dbRootExtentTrackers.size(); i++) + { + tmpExtentInfo = dbRootExtentTrackers[i]->getDBRootExtentList(); + colStructList[i].fColPartition = tmpExtentInfo[currentDBrootIdx].fPartition; + colStructList[i].fColSegment = tmpExtentInfo[currentDBrootIdx].fSegment; + colStructList[i].fColDbRoot = tmpExtentInfo[currentDBrootIdx].fDbRoot; + //cout << "Load from dbrootExtenttracker oid:dbroot:part:seg = " <getColExtsInfo(colStructList[i].dataOid); + ColExtsInfo::iterator it = aColExtsInfo.begin(); + while (it != aColExtsInfo.end()) + { + if ((it->dbRoot == colStructList[i].fColDbRoot) && (it->partNum == colStructList[i].fColPartition) && (it->segNum == colStructList[i].fColSegment)) + break; + it++; + } + + if (it == aColExtsInfo.end()) //add this one to the list + { + ColExtInfo aExt; + aExt.dbRoot = colStructList[i].fColDbRoot; + aExt.partNum = colStructList[i].fColPartition; + aExt.segNum = colStructList[i].fColSegment; + aExt.compType = colStructList[i].fCompressionType; + aExt.isDict = false; + if (bFirstExtentOnThisPM) + { + aExt.hwm = extents[i].startBlkOffset; + aExt.isNewExt = true; + //cout << "adding a ext to metadata" << endl; + } + else + { + std::vector tmpExtentInfo; + tmpExtentInfo = dbRootExtentTrackers[i]->getDBRootExtentList(); + aExt.isNewExt = false; + aExt.hwm = tmpExtentInfo[currentDBrootIdx].fLocalHwm; + //cout << "oid " << colStructList[i].dataOid << " gets hwm " << aExt.hwm << endl; + } + aExt.current = true; + aColExtsInfo.push_back(aExt); + //cout << "get from extentinfo oid:hwm = " << colStructList[i].dataOid << ":" << aExt.hwm << endl; + } + + tableMetaData->setColExtsInfo(colStructList[i].dataOid, aColExtsInfo); + } + + for (i=0; i < dctnryStructList.size(); i++) + { + if (dctnryStructList[i].dctnryOid > 0) + { + ColExtsInfo aColExtsInfo = tableMetaData->getColExtsInfo(dctnryStructList[i].dctnryOid); + ColExtsInfo::iterator it = aColExtsInfo.begin(); + while (it != aColExtsInfo.end()) + { + if ((it->dbRoot == dctnryStructList[i].fColDbRoot) && (it->partNum == dctnryStructList[i].fColPartition) && (it->segNum == dctnryStructList[i].fColSegment)) + break; + it++; + } + + if (it == aColExtsInfo.end()) //add this one to the list + { + ColExtInfo aExt; + aExt.dbRoot = dctnryStructList[i].fColDbRoot; + aExt.partNum = dctnryStructList[i].fColPartition; + aExt.segNum = dctnryStructList[i].fColSegment; + aExt.compType = dctnryStructList[i].fCompressionType; + aExt.isDict = true; + aColExtsInfo.push_back(aExt); + } + tableMetaData->setColExtsInfo(dctnryStructList[i].dctnryOid, aColExtsInfo); + } + } + + } // if (isFirstBatchPm) + else //get the extent info from tableMetaData + { + ColExtsInfo aColExtsInfo = tableMetaData->getColExtsInfo(colStructList[0].dataOid); + ColExtsInfo::iterator it = aColExtsInfo.begin(); + while (it != aColExtsInfo.end()) + { + if (it->current) + break; + it++; + } + if (it == aColExtsInfo.end()) + return 1; + + for (i=0; i < colStructList.size(); i++) + { + colStructList[i].fColPartition = it->partNum; + colStructList[i].fColSegment = it->segNum; + colStructList[i].fColDbRoot = it->dbRoot; + dctnryStructList[i].fColPartition = it->partNum; + dctnryStructList[i].fColSegment = it->segNum; + dctnryStructList[i].fColDbRoot = it->dbRoot; + } + } + + totalColumns = colStructList.size(); + totalRow = colValueList.size() / totalColumns; + rowIdArray = new RID[totalRow]; + // use scoped_array to ensure ptr deletion regardless of where we return + boost::scoped_array rowIdArrayPtr(rowIdArray); + memset(rowIdArray, 0, (sizeof(RID)*totalRow)); + + //-------------------------------------------------------------------------- + // allocate row id(s) + //-------------------------------------------------------------------------- + curColStruct = colStructList[0]; + colOp = m_colOp[op(curColStruct.fCompressionType)]; + + colOp->initColumn(curCol); + + //Get the correct segment, partition, column file + vector colExtentInfo; //Save those empty extents in case of failure to rollback + vector dictExtentInfo; //Save those empty extents in case of failure to rollback + vector fileInfo; + dbRoot = curColStruct.fColDbRoot; + //use the first column to calculate row id + ColExtsInfo aColExtsInfo = tableMetaData->getColExtsInfo(colStructList[0].dataOid); + ColExtsInfo::iterator it = aColExtsInfo.begin(); + while (it != aColExtsInfo.end()) + { + if ((it->dbRoot == colStructList[0].fColDbRoot) && (it->partNum == colStructList[0].fColPartition) && (it->segNum == colStructList[0].fColSegment) && it->current ) + break; + it++; + } + if (it != aColExtsInfo.end()) + { + hwm = it->hwm; + //cout << "Got from colextinfo hwm for oid " << colStructList[0].dataOid << " is " << hwm << " and seg is " << colStructList[0].fColSegment << endl; + } + + oldHwm = hwm; //Save this info for rollback + //need to pass real dbRoot, partition, and segment to setColParam + colOp->setColParam(curCol, 0, curColStruct.colWidth, curColStruct.colDataType, + curColStruct.colType, curColStruct.dataOid, curColStruct.fCompressionType, + curColStruct.fColDbRoot, curColStruct.fColPartition, curColStruct.fColSegment); + rc = colOp->openColumnFile(curCol, segFile, useTmpSuffix); // @bug 5572 HDFS tmp file + if (rc != NO_ERROR) { + return rc; + } + + //get hwm first + // @bug 286 : fix for bug 286 - correct the typo in getHWM + //RETURN_ON_ERROR(BRMWrapper::getInstance()->getHWM(curColStruct.dataOid, hwm)); + + Column newCol; + +#ifdef PROFILE +timer.start("allocRowId"); +#endif + newColStructList = colStructList; + newDctnryStructList = dctnryStructList; + bool bUseStartExtent = true; + if (idbdatafile::IDBPolicy::useHdfs()) + insertSelect = true; + + rc = colOp->allocRowId(txnid, bUseStartExtent, + curCol, (uint64_t)totalRow, rowIdArray, hwm, newExtent, rowsLeft, newHwm, newFile, + newColStructList, newDctnryStructList, dbRootExtentTrackers, insertSelect, true, tableOid, isFirstBatchPm); + + //cout << "after allocrowid, total row = " < 256K. + // if totalRow == rowsLeft, then not adding rows to 1st extent, so skip it. + //-------------------------------------------------------------------------- +// DMC-SHARED_NOTHING_NOTE: Is it safe to assume only part0 seg0 is abbreviated? + if ((curCol.dataFile.fPartition == 0) && + (curCol.dataFile.fSegment == 0) && + ((totalRow-rowsLeft) > 0) && + (rowIdArray[totalRow-rowsLeft-1] >= (RID)INITIAL_EXTENT_ROWS_TO_DISK)) + { + for (unsigned k=1; ksetColParam(expandCol, 0, + colStructList[k].colWidth, + colStructList[k].colDataType, + colStructList[k].colType, + colStructList[k].dataOid, + colStructList[k].fCompressionType, + colStructList[k].fColDbRoot, + colStructList[k].fColPartition, + colStructList[k].fColSegment); + rc = colOp->openColumnFile(expandCol, segFile, true); // @bug 5572 HDFS tmp file + if (rc == NO_ERROR) + { + if (colOp->abbreviatedExtent(expandCol.dataFile.pFile, colStructList[k].colWidth)) + { + rc = colOp->expandAbbrevExtent(expandCol); + } + } + if (rc != NO_ERROR) + { + return rc; + } + colOp->closeColumnFile(expandCol); + } + } + + //-------------------------------------------------------------------------- + // Tokenize data if needed + //-------------------------------------------------------------------------- + if (insertSelect && isAutoCommitOn) + BRMWrapper::setUseVb( false ); + else + BRMWrapper::setUseVb( true ); + dictStr::iterator dctStr_iter; + uint64_t *colValPtr; + size_t rowsPerColumn = colValueList.size() / colStructList.size(); + for (i = 0; i < colStructList.size(); i++) + { + if (colStructList[i].tokenFlag) + { + dctStr_iter = dictStrList[i].begin(); + Dctnry* dctnry = m_dctnry[op(dctnryStructList[i].fCompressionType)]; + rc = dctnry->openDctnry(dctnryStructList[i].dctnryOid, + dctnryStructList[i].fColDbRoot, dctnryStructList[i].fColPartition, + dctnryStructList[i].fColSegment, + useTmpSuffix); // @bug 5572 HDFS tmp file + if (rc !=NO_ERROR) + { + cout << "Error opening dctnry file " << dctnryStructList[i].dctnryOid<< endl; + return rc; + } + + for (uint32_t rows = 0; rows < (totalRow - rowsLeft); rows++) + { + colValPtr = &colValueList[(i*rowsPerColumn) + rows]; + if (dctStr_iter->length() == 0) + { + Token nullToken; + memcpy(colValPtr, &nullToken, 8); + } + else + { +#ifdef PROFILE +timer.start("tokenize"); +#endif + DctnryTuple dctTuple; + dctTuple.sigValue = (unsigned char*)dctStr_iter->c_str(); + dctTuple.sigSize = dctStr_iter->length(); + dctTuple.isNull = false; + rc = tokenize(txnid, dctTuple, dctnryStructList[i].fCompressionType); + if (rc != NO_ERROR) + { + dctnry->closeDctnry(); + return rc; + } +#ifdef PROFILE +timer.stop("tokenize"); +#endif + memcpy(colValPtr, &dctTuple.token, 8); + } + dctStr_iter++; + + } + //close dictionary files + rc = dctnry->closeDctnry(false); + if (rc != NO_ERROR) + return rc; + + if (newExtent) + { + //@Bug 4854 back up hwm chunk for the file to be modified + if (fRBMetaWriter) + fRBMetaWriter->backupDctnryHWMChunk(newDctnryStructList[i].dctnryOid, newDctnryStructList[i].fColDbRoot, newDctnryStructList[i].fColPartition, newDctnryStructList[i].fColSegment); + rc = dctnry->openDctnry(newDctnryStructList[i].dctnryOid, + newDctnryStructList[i].fColDbRoot, newDctnryStructList[i].fColPartition, + newDctnryStructList[i].fColSegment, + false); // @bug 5572 HDFS tmp file + if (rc !=NO_ERROR) + return rc; + + for (uint32_t rows = 0; rows < rowsLeft; rows++) + { + colValPtr = &colValueList[(i*rowsPerColumn) + rows]; + if (dctStr_iter->length() == 0) + { + Token nullToken; + memcpy(colValPtr, &nullToken, 8); + } + else + { +#ifdef PROFILE +timer.start("tokenize"); +#endif + DctnryTuple dctTuple; + dctTuple.sigValue = (unsigned char*)dctStr_iter->c_str(); + dctTuple.sigSize = dctStr_iter->length(); + dctTuple.isNull = false; + rc = tokenize(txnid, dctTuple, newDctnryStructList[i].fCompressionType); + if (rc != NO_ERROR) + { + dctnry->closeDctnry(); + return rc; + } +#ifdef PROFILE +timer.stop("tokenize"); +#endif + memcpy(colValPtr, &dctTuple.token, 8); + } + dctStr_iter++; + } + //close dictionary files + rc = dctnry->closeDctnry(false); + if (rc != NO_ERROR) + return rc; + } + } + } + if (insertSelect && isAutoCommitOn) + BRMWrapper::setUseVb( false ); + else + BRMWrapper::setUseVb( true ); + + //-------------------------------------------------------------------------- + // Update column info structure @Bug 1862 set hwm, and + // Prepare ValueList for new extent (if applicable) + //-------------------------------------------------------------------------- + //@Bug 2205 Check whether all rows go to the new extent + RID lastRid = 0; + RID lastRidNew = 0; + if (totalRow-rowsLeft > 0) + { + lastRid = rowIdArray[totalRow-rowsLeft-1]; + lastRidNew = rowIdArray[totalRow-1]; + } + else + { + lastRid = 0; + lastRidNew = rowIdArray[totalRow-1]; + } + //cout << "rowid allocated is " << lastRid << endl; + //if a new extent is created, all the columns in this table should have their own new extent + //First column already processed + + //@Bug 1701. Close the file (if uncompressed) + m_colOp[op(curCol.compressionType)]->closeColumnFile(curCol); + //cout << "Saving hwm info for new ext batch" << endl; + //Update hwm to set them in the end + bool succFlag = false; + unsigned colWidth = 0; + int curFbo = 0, curBio; + for (i=0; i < totalColumns; i++) + { + //shoud be obtained from saved hwm + aColExtsInfo = tableMetaData->getColExtsInfo(colStructList[i].dataOid); + it = aColExtsInfo.begin(); + while (it != aColExtsInfo.end()) + { + if ((it->dbRoot == colStructList[i].fColDbRoot) && (it->partNum == colStructList[i].fColPartition) + && (it->segNum == colStructList[i].fColSegment) && it->current) + break; + it++; + } + if (it != aColExtsInfo.end()) //update hwm info + { + oldHwm = it->hwm; + } + + // save hwm for the old extent + colWidth = colStructList[i].colWidth; + succFlag = colOp->calculateRowId(lastRid, BYTE_PER_BLOCK/colWidth, colWidth, curFbo, curBio); + //cout << "insertcolumnrec oid:rid:fbo:oldhwm = " << colStructList[i].dataOid << ":" << lastRid << ":" << curFbo << ":" << oldHwm << endl; + if (succFlag) + { + if ((HWM)curFbo >= oldHwm) + { + it->hwm = (HWM)curFbo; + } + //@Bug 4947. set current to false for old extent. + if (newExtent) + { + it->current = false; + } + + //cout << "updated old ext info for oid " << colStructList[i].dataOid << " dbroot:part:seg:hwm:current = " + //<< it->dbRoot<<":"<partNum<<":"<segNum<<":"<hwm<<":"<< it->current<< " and newExtent is " << newExtent << endl; + } + else + return ERR_INVALID_PARAM; + + //update hwm for the new extent + if (newExtent) + { + it = aColExtsInfo.begin(); + while (it != aColExtsInfo.end()) + { + if ((it->dbRoot == newColStructList[i].fColDbRoot) && (it->partNum == newColStructList[i].fColPartition) + && (it->segNum == newColStructList[i].fColSegment) && it->current) + break; + it++; + } + succFlag = colOp->calculateRowId(lastRidNew, BYTE_PER_BLOCK/colWidth, colWidth, curFbo, curBio); + if (succFlag) + { + if (it != aColExtsInfo.end()) + { + it->hwm = (HWM)curFbo; + //cout << "setting hwm to " << (int)curFbo <<" for seg " <segNum << endl; + it->current = true; + } + } + else + return ERR_INVALID_PARAM; + } + tableMetaData->setColExtsInfo(colStructList[i].dataOid, aColExtsInfo); + } + + // end of allocate row id + +#ifdef PROFILE +timer.start("writeColumnRec"); +#endif +//cout << "Writing column record" << endl; + + if (rc == NO_ERROR) + { + //---------------------------------------------------------------------- + //Mark extents invalid + //---------------------------------------------------------------------- + vector lbids; + vector colDataTypes; + bool successFlag = true; + unsigned width = 0; + int curFbo = 0, curBio, lastFbo = -1; + + if (isFirstBatchPm && (totalRow == rowsLeft)) + {} + else { + for (unsigned i = 0; i < colStructList.size(); i++) + { + colOp = m_colOp[op(colStructList[i].fCompressionType)]; + width = colStructList[i].colWidth; + successFlag = colOp->calculateRowId(lastRid , BYTE_PER_BLOCK/width, width, curFbo, curBio); + if (successFlag) { + if (curFbo != lastFbo) { + RETURN_ON_ERROR(AddLBIDtoList(txnid, + lbids, + colDataTypes, + colStructList[i], + curFbo)); + } + } + } + } + + if (lbids.size() > 0) + rc = BRMWrapper::getInstance()->markExtentsInvalid(lbids, colDataTypes); + + //---------------------------------------------------------------------- + // Write row(s) to database file(s) + //---------------------------------------------------------------------- + rc = writeColumnRecBinary(txnid, colStructList, colValueList, rowIdArray, newColStructList, tableOid, useTmpSuffix); // @bug 5572 HDFS tmp file + } + return rc; +} + + int WriteEngineWrapper::insertColumnRec_SYS(const TxnID& txnid, ColStructList& colStructList, ColValueList& colValueList, @@ -3909,6 +4541,204 @@ timer.finish(); return rc; } +int WriteEngineWrapper::writeColumnRecBinary(const TxnID& txnid, + const ColStructList& colStructList, + std::vector& colValueList, + RID* rowIdArray, + const ColStructList& newColStructList, + const int32_t tableOid, + bool useTmpSuffix, + bool versioning) +{ + bool bExcp; + int rc = 0; + void* valArray; + string segFile; + Column curCol; + ColStructList::size_type totalColumn; + ColStructList::size_type i; + size_t totalRow; + + setTransId(txnid); + + totalColumn = colStructList.size(); +#ifdef PROFILE +StopWatch timer; +#endif + totalRow = colValueList.size() / totalColumn; + + valArray = malloc(sizeof(uint64_t) * totalRow); + + if (totalRow == 0) + return rc; + + TableMetaData* aTbaleMetaData = TableMetaData::makeTableMetaData(tableOid); + for (i = 0; i < totalColumn; i++) + { + RID * secondPart = rowIdArray + totalRow; + //@Bug 2205 Check if all rows go to the new extent + //Write the first batch + RID * firstPart = rowIdArray; + ColumnOp* colOp = m_colOp[op(colStructList[i].fCompressionType)]; + + // set params + colOp->initColumn(curCol); + // need to pass real dbRoot, partition, and segment to setColParam + colOp->setColParam(curCol, 0, colStructList[i].colWidth, + colStructList[i].colDataType, colStructList[i].colType, colStructList[i].dataOid, + colStructList[i].fCompressionType, colStructList[i].fColDbRoot, + colStructList[i].fColPartition, colStructList[i].fColSegment); + + ColExtsInfo aColExtsInfo = aTbaleMetaData->getColExtsInfo(colStructList[i].dataOid); + ColExtsInfo::iterator it = aColExtsInfo.begin(); + while (it != aColExtsInfo.end()) + { + if ((it->dbRoot == colStructList[i].fColDbRoot) && (it->partNum == colStructList[i].fColPartition) && (it->segNum == colStructList[i].fColSegment)) + break; + it++; + } + + if (it == aColExtsInfo.end()) //add this one to the list + { + ColExtInfo aExt; + aExt.dbRoot =colStructList[i].fColDbRoot; + aExt.partNum = colStructList[i].fColPartition; + aExt.segNum = colStructList[i].fColSegment; + aExt.compType = colStructList[i].fCompressionType; + aColExtsInfo.push_back(aExt); + aTbaleMetaData->setColExtsInfo(colStructList[i].dataOid, aColExtsInfo); + } + + rc = colOp->openColumnFile(curCol, segFile, useTmpSuffix, IO_BUFF_SIZE); // @bug 5572 HDFS tmp file + if (rc != NO_ERROR) + break; + + // handling versioning + vector rangeList; + if (versioning) + { + rc = processVersionBuffer(curCol.dataFile.pFile, txnid, colStructList[i], + colStructList[i].colWidth, totalRow, firstPart, rangeList); + if (rc != NO_ERROR) { + if (colStructList[i].fCompressionType == 0) + { + curCol.dataFile.pFile->flush(); + } + + BRMWrapper::getInstance()->writeVBEnd(txnid, rangeList); + break; + } + } + + //totalRow1 -= totalRow2; + // have to init the size here + // nullArray = (bool*) malloc(sizeof(bool) * totalRow); + uint8_t tmp8; + uint16_t tmp16; + uint32_t tmp32; + float tmpF; + double tmpD; + for (size_t j = 0; j < totalRow; j++) + { + uint64_t curValue = colValueList[(totalRow*i) + j]; + switch (colStructList[i].colType) + { + case WriteEngine::WR_INT: + tmp32 = curValue; + ((int*)valArray)[j] = tmp32; + break; + case WriteEngine::WR_UINT: + tmp32 = curValue; + ((uint32_t*)valArray)[j] = tmp32; + break; + case WriteEngine::WR_VARBINARY : // treat same as char for now + case WriteEngine::WR_CHAR: + case WriteEngine::WR_BLOB: + case WriteEngine::WR_TEXT: + switch (colStructList[i].colWidth) + { + case 1: + tmp8 = curValue; + ((uint8_t*)valArray)[j] = tmp8; + break; + case 2: + tmp16 = curValue; + ((uint16_t*)valArray)[j] = tmp16; + break; + case 3: + tmp32 = curValue; + ((uint32_t*)valArray)[j] = tmp32; + break; + case 4: + ((uint32_t*)valArray)[j] = curValue; + break; + } + + break; + case WriteEngine::WR_FLOAT: + tmp32 = curValue; + memcpy(&((float*)valArray)[j], &tmp32, 4); + break; + case WriteEngine::WR_DOUBLE: + memcpy(&((double*)valArray)[j], &curValue, 8); + break; + case WriteEngine::WR_BYTE: + tmp8 = curValue; + ((char*)valArray)[j] = tmp8; + break; + case WriteEngine::WR_UBYTE: + tmp8 = curValue; + ((uint8_t*)valArray)[j] = tmp8; + break; + case WriteEngine::WR_SHORT: + tmp16 = curValue; + ((short*)valArray)[j] = tmp16; + break; + case WriteEngine::WR_USHORT: + tmp32 = curValue; + ((uint16_t*)valArray)[j] = tmp16; + break; + case WriteEngine::WR_LONGLONG: + tmp32 = curValue; + ((long long*)valArray)[j] = tmp32; + break; + case WriteEngine::WR_ULONGLONG: + ((uint64_t*)valArray)[j] = curValue; + break; + case WriteEngine::WR_TOKEN: + memcpy(&((Token*)valArray)[j], &curValue, 8); + break; + } + } + + +#ifdef PROFILE +timer.start("writeRow "); +#endif + rc = colOp->writeRow(curCol, totalRow, firstPart, valArray); +#ifdef PROFILE +timer.stop("writeRow "); +#endif + colOp->closeColumnFile(curCol); + + if (versioning) + BRMWrapper::getInstance()->writeVBEnd(txnid, rangeList); + + // check error + if (rc != NO_ERROR) + break; + + } // end of for (i = 0 + if (valArray != NULL) + free(valArray); + +#ifdef PROFILE +timer.finish(); +#endif + return rc; +} + + int WriteEngineWrapper::writeColumnRec(const TxnID& txnid, const ColStructList& colStructList, const ColValueList& colValueList, diff --git a/writeengine/wrapper/writeengine.h b/writeengine/wrapper/writeengine.h index a47b9e291..7c7862d5f 100644 --- a/writeengine/wrapper/writeengine.h +++ b/writeengine/wrapper/writeengine.h @@ -303,6 +303,20 @@ public: bool isAutoCommitOn = false, OID tableOid = 0, bool isFirstBatchPm = false); + + EXPORT int insertColumnRecsBinary(const TxnID& txnid, + ColStructList& colStructList, + std::vector& colValueList, + DctnryStructList& dctnryStructList, + DictStrList& dictStrList, + std::vector > & dbRootExtentTrackers, + RBMetaWriter* fRBMetaWriter, + bool bFirstExtentOnThisPM, + bool insertSelect = false, + bool isAutoCommitOn = false, + OID tableOid = 0, + bool isFirstBatchPm = false); + /** * @brief Insert values into systables @@ -646,6 +660,11 @@ private: ColValueList& newColValueList, const int32_t tableOid, bool useTmpSuffix, bool versioning = true); + int writeColumnRecBinary(const TxnID& txnid, const ColStructList& colStructList, + std::vector& colValueList, + RID* rowIdArray, const ColStructList& newColStructList, + const int32_t tableOid, + bool useTmpSuffix, bool versioning = true); //@Bug 1886,2870 pass the address of ridList vector